2024-11-08 00:14:23,248 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-08 00:14:23,273 main DEBUG Took 0.016472 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 00:14:23,276 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 00:14:23,277 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 00:14:23,277 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 00:14:23,279 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,289 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 00:14:23,331 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,334 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,335 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,338 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,340 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,341 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,341 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,342 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,344 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,344 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,345 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,345 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,346 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,347 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,348 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,348 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,349 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,349 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:14:23,352 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,353 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 00:14:23,355 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:14:23,356 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 00:14:23,359 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 00:14:23,360 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 00:14:23,361 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 00:14:23,362 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 00:14:23,377 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 00:14:23,381 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 00:14:23,385 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 00:14:23,386 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 00:14:23,387 main DEBUG createAppenders(={Console}) 2024-11-08 00:14:23,400 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-08 00:14:23,401 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-08 00:14:23,401 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-08 00:14:23,402 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 00:14:23,402 main DEBUG OutputStream closed 2024-11-08 00:14:23,403 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 00:14:23,403 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 00:14:23,404 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-08 00:14:23,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 00:14:23,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 00:14:23,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 00:14:23,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 00:14:23,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 00:14:23,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 00:14:23,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 00:14:23,532 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 00:14:23,532 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 00:14:23,533 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 00:14:23,533 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 00:14:23,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 00:14:23,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 00:14:23,535 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 00:14:23,535 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 00:14:23,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 00:14:23,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 00:14:23,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 00:14:23,540 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 00:14:23,541 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-08 00:14:23,541 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 00:14:23,542 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-08T00:14:23,560 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-08 00:14:23,564 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 00:14:23,564 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T00:14:23,939 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a 2024-11-08T00:14:23,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T00:14:23,991 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2, deleteOnExit=true 2024-11-08T00:14:23,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T00:14:23,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/test.cache.data in system properties and HBase conf 2024-11-08T00:14:23,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T00:14:23,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.log.dir in system properties and HBase conf 2024-11-08T00:14:23,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T00:14:23,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T00:14:23,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T00:14:24,138 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T00:14:24,275 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:14:24,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T00:14:24,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T00:14:24,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T00:14:24,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T00:14:24,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T00:14:24,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T00:14:24,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T00:14:24,300 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T00:14:24,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T00:14:24,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/nfs.dump.dir in system properties and HBase conf 2024-11-08T00:14:24,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/java.io.tmpdir in system properties and HBase conf 2024-11-08T00:14:24,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T00:14:24,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T00:14:24,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T00:14:25,912 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T00:14:26,073 INFO [Time-limited test {}] log.Log(170): Logging initialized @3895ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T00:14:26,221 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T00:14:26,346 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T00:14:26,414 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T00:14:26,414 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T00:14:26,417 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T00:14:26,443 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T00:14:26,463 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.log.dir/,AVAILABLE} 2024-11-08T00:14:26,464 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T00:14:26,799 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/java.io.tmpdir/jetty-localhost-43153-hadoop-hdfs-3_4_1-tests_jar-_-any-1466528536590146539/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T00:14:26,821 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:43153} 2024-11-08T00:14:26,821 INFO [Time-limited test {}] server.Server(415): Started @4644ms 2024-11-08T00:14:27,824 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T00:14:27,836 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T00:14:27,844 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T00:14:27,845 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T00:14:27,845 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T00:14:27,857 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.log.dir/,AVAILABLE} 2024-11-08T00:14:27,858 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T00:14:27,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/java.io.tmpdir/jetty-localhost-37611-hadoop-hdfs-3_4_1-tests_jar-_-any-3952269767440054857/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T00:14:27,992 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:37611} 2024-11-08T00:14:27,993 INFO [Time-limited test {}] server.Server(415): Started @5816ms 2024-11-08T00:14:28,064 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T00:14:28,334 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T00:14:28,343 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T00:14:28,364 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T00:14:28,365 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T00:14:28,365 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T00:14:28,385 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.log.dir/,AVAILABLE} 2024-11-08T00:14:28,386 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T00:14:28,560 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/java.io.tmpdir/jetty-localhost-41951-hadoop-hdfs-3_4_1-tests_jar-_-any-624621567251688553/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T00:14:28,562 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:41951} 2024-11-08T00:14:28,562 INFO [Time-limited test {}] server.Server(415): Started @6385ms 2024-11-08T00:14:28,572 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T00:14:28,707 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T00:14:28,739 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T00:14:28,746 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T00:14:28,746 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T00:14:28,747 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T00:14:28,749 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.log.dir/,AVAILABLE} 2024-11-08T00:14:28,750 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T00:14:28,892 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/java.io.tmpdir/jetty-localhost-32955-hadoop-hdfs-3_4_1-tests_jar-_-any-13645789693765963209/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T00:14:28,894 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:32955} 2024-11-08T00:14:28,894 INFO [Time-limited test {}] server.Server(415): Started @6717ms 2024-11-08T00:14:28,896 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T00:14:29,909 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data2/current/BP-76941553-172.17.0.3-1731024865284/current, will proceed with Du for space computation calculation, 2024-11-08T00:14:29,909 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data1/current/BP-76941553-172.17.0.3-1731024865284/current, will proceed with Du for space computation calculation, 2024-11-08T00:14:29,983 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T00:14:30,073 WARN [Thread-131 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data3/current/BP-76941553-172.17.0.3-1731024865284/current, will proceed with Du for space computation calculation, 2024-11-08T00:14:30,079 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data4/current/BP-76941553-172.17.0.3-1731024865284/current, will proceed with Du for space computation calculation, 2024-11-08T00:14:30,096 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f7db480f37ce28b with lease ID 0x1f8bf7a42cb9afa1: Processing first storage report for DS-697ab831-8e11-49a9-ae3f-5d4889ea6c9b from datanode DatanodeRegistration(127.0.0.1:44189, datanodeUuid=d75300ff-4353-4e3c-98b5-6111bd901f5b, infoPort=44713, infoSecurePort=0, ipcPort=39401, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284) 2024-11-08T00:14:30,097 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f7db480f37ce28b with lease ID 0x1f8bf7a42cb9afa1: from storage DS-697ab831-8e11-49a9-ae3f-5d4889ea6c9b node DatanodeRegistration(127.0.0.1:44189, datanodeUuid=d75300ff-4353-4e3c-98b5-6111bd901f5b, infoPort=44713, infoSecurePort=0, ipcPort=39401, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-08T00:14:30,101 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f7db480f37ce28b with lease ID 0x1f8bf7a42cb9afa1: Processing first storage report for DS-a7b8d159-a4c9-46e0-ad66-59f9ce012ff7 from datanode DatanodeRegistration(127.0.0.1:44189, datanodeUuid=d75300ff-4353-4e3c-98b5-6111bd901f5b, infoPort=44713, infoSecurePort=0, ipcPort=39401, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284) 2024-11-08T00:14:30,101 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f7db480f37ce28b with lease ID 0x1f8bf7a42cb9afa1: from storage DS-a7b8d159-a4c9-46e0-ad66-59f9ce012ff7 node DatanodeRegistration(127.0.0.1:44189, datanodeUuid=d75300ff-4353-4e3c-98b5-6111bd901f5b, infoPort=44713, infoSecurePort=0, ipcPort=39401, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T00:14:30,149 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T00:14:30,158 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1552d43d745eea9 with lease ID 0x1f8bf7a42cb9afa2: Processing first storage report for DS-05680217-d0db-4d7d-a701-2d3460612161 from datanode DatanodeRegistration(127.0.0.1:36957, datanodeUuid=82969024-0613-43db-8af9-2144424c9c8e, infoPort=39279, infoSecurePort=0, ipcPort=36737, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284) 2024-11-08T00:14:30,158 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1552d43d745eea9 with lease ID 0x1f8bf7a42cb9afa2: from storage DS-05680217-d0db-4d7d-a701-2d3460612161 node DatanodeRegistration(127.0.0.1:36957, datanodeUuid=82969024-0613-43db-8af9-2144424c9c8e, infoPort=39279, infoSecurePort=0, ipcPort=36737, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T00:14:30,159 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1552d43d745eea9 with lease ID 0x1f8bf7a42cb9afa2: Processing first storage report for DS-f839087d-93a9-4ef1-863c-9f4426935d90 from datanode DatanodeRegistration(127.0.0.1:36957, datanodeUuid=82969024-0613-43db-8af9-2144424c9c8e, infoPort=39279, infoSecurePort=0, ipcPort=36737, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284) 2024-11-08T00:14:30,159 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1552d43d745eea9 with lease ID 0x1f8bf7a42cb9afa2: from storage DS-f839087d-93a9-4ef1-863c-9f4426935d90 node DatanodeRegistration(127.0.0.1:36957, datanodeUuid=82969024-0613-43db-8af9-2144424c9c8e, infoPort=39279, infoSecurePort=0, ipcPort=36737, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T00:14:30,275 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data5/current/BP-76941553-172.17.0.3-1731024865284/current, will proceed with Du for space computation calculation, 2024-11-08T00:14:30,291 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data6/current/BP-76941553-172.17.0.3-1731024865284/current, will proceed with Du for space computation calculation, 2024-11-08T00:14:30,333 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T00:14:30,342 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3473a6460d751ce8 with lease ID 0x1f8bf7a42cb9afa3: Processing first storage report for DS-c7c86592-a2a6-4354-ab71-9748dcc4c027 from datanode DatanodeRegistration(127.0.0.1:36391, datanodeUuid=89c99946-75ec-4aaf-bc46-cfe5f0ddb9fc, infoPort=43187, infoSecurePort=0, ipcPort=43797, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284) 2024-11-08T00:14:30,343 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3473a6460d751ce8 with lease ID 0x1f8bf7a42cb9afa3: from storage DS-c7c86592-a2a6-4354-ab71-9748dcc4c027 node DatanodeRegistration(127.0.0.1:36391, datanodeUuid=89c99946-75ec-4aaf-bc46-cfe5f0ddb9fc, infoPort=43187, infoSecurePort=0, ipcPort=43797, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T00:14:30,343 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3473a6460d751ce8 with lease ID 0x1f8bf7a42cb9afa3: Processing first storage report for DS-b44d5375-2695-47ca-9583-5cda11b45782 from datanode DatanodeRegistration(127.0.0.1:36391, datanodeUuid=89c99946-75ec-4aaf-bc46-cfe5f0ddb9fc, infoPort=43187, infoSecurePort=0, ipcPort=43797, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284) 2024-11-08T00:14:30,344 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3473a6460d751ce8 with lease ID 0x1f8bf7a42cb9afa3: from storage DS-b44d5375-2695-47ca-9583-5cda11b45782 node DatanodeRegistration(127.0.0.1:36391, datanodeUuid=89c99946-75ec-4aaf-bc46-cfe5f0ddb9fc, infoPort=43187, infoSecurePort=0, ipcPort=43797, storageInfo=lv=-57;cid=testClusterID;nsid=1738505688;c=1731024865284), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T00:14:30,412 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a 2024-11-08T00:14:30,498 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/zookeeper_0, clientPort=54358, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T00:14:30,509 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54358 2024-11-08T00:14:30,524 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:30,528 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:30,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741825_1001 (size=7) 2024-11-08T00:14:30,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741825_1001 (size=7) 2024-11-08T00:14:30,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741825_1001 (size=7) 2024-11-08T00:14:30,876 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 with version=8 2024-11-08T00:14:30,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/hbase-staging 2024-11-08T00:14:30,988 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T00:14:31,232 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3302f0f507bd:0 server-side Connection retries=45 2024-11-08T00:14:31,242 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:31,242 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:31,248 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:14:31,248 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:31,248 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:14:31,388 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T00:14:31,454 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T00:14:31,465 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T00:14:31,469 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:14:31,499 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3736 (auto-detected) 2024-11-08T00:14:31,500 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-08T00:14:31,521 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36429 2024-11-08T00:14:31,545 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36429 connecting to ZooKeeper ensemble=127.0.0.1:54358 2024-11-08T00:14:31,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:364290x0, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:14:31,897 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36429-0x10117cb57f40000 connected 2024-11-08T00:14:32,607 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:32,613 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:32,625 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:14:32,631 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20, hbase.cluster.distributed=false 2024-11-08T00:14:32,667 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:14:32,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36429 2024-11-08T00:14:32,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36429 2024-11-08T00:14:32,676 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36429 2024-11-08T00:14:32,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36429 2024-11-08T00:14:32,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36429 2024-11-08T00:14:32,799 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3302f0f507bd:0 server-side Connection retries=45 2024-11-08T00:14:32,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:32,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:32,801 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:14:32,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:32,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:14:32,804 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T00:14:32,808 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:14:32,810 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38371 2024-11-08T00:14:32,813 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38371 connecting to ZooKeeper ensemble=127.0.0.1:54358 2024-11-08T00:14:32,815 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:32,821 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:32,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:383710x0, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:14:32,837 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:383710x0, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:14:32,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38371-0x10117cb57f40001 connected 2024-11-08T00:14:32,841 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T00:14:32,854 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T00:14:32,858 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T00:14:32,863 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:14:32,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38371 2024-11-08T00:14:32,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38371 2024-11-08T00:14:32,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38371 2024-11-08T00:14:32,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38371 2024-11-08T00:14:32,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38371 2024-11-08T00:14:32,884 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3302f0f507bd:0 server-side Connection retries=45 2024-11-08T00:14:32,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:32,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:32,884 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:14:32,885 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:32,885 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:14:32,885 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T00:14:32,885 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:14:32,886 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44033 2024-11-08T00:14:32,888 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44033 connecting to ZooKeeper ensemble=127.0.0.1:54358 2024-11-08T00:14:32,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:32,891 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:32,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440330x0, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:14:32,899 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44033-0x10117cb57f40002 connected 2024-11-08T00:14:32,899 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:14:32,900 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T00:14:32,901 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T00:14:32,903 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T00:14:32,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:14:32,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44033 2024-11-08T00:14:32,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44033 2024-11-08T00:14:32,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44033 2024-11-08T00:14:32,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44033 2024-11-08T00:14:32,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44033 2024-11-08T00:14:32,926 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3302f0f507bd:0 server-side Connection retries=45 2024-11-08T00:14:32,926 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:32,926 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:32,927 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:14:32,927 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:14:32,927 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:14:32,927 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T00:14:32,927 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:14:32,929 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36139 2024-11-08T00:14:32,930 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36139 connecting to ZooKeeper ensemble=127.0.0.1:54358 2024-11-08T00:14:32,932 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:32,936 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:32,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361390x0, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:14:32,952 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:361390x0, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:14:32,953 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T00:14:32,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36139-0x10117cb57f40003 connected 2024-11-08T00:14:32,961 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T00:14:32,962 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T00:14:32,964 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:14:32,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36139 2024-11-08T00:14:32,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36139 2024-11-08T00:14:32,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36139 2024-11-08T00:14:32,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36139 2024-11-08T00:14:32,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36139 2024-11-08T00:14:33,001 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3302f0f507bd:36429 2024-11-08T00:14:33,003 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3302f0f507bd,36429,1731024871043 2024-11-08T00:14:33,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:14:33,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:14:33,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:14:33,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:14:33,019 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3302f0f507bd,36429,1731024871043 2024-11-08T00:14:33,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T00:14:33,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T00:14:33,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T00:14:33,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,061 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T00:14:33,062 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3302f0f507bd,36429,1731024871043 from backup master directory 2024-11-08T00:14:33,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:14:33,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:14:33,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3302f0f507bd,36429,1731024871043 2024-11-08T00:14:33,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:14:33,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:14:33,077 WARN [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:14:33,077 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3302f0f507bd,36429,1731024871043 2024-11-08T00:14:33,079 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T00:14:33,081 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T00:14:33,139 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/hbase.id] with ID: 2dc56221-06a4-403e-ac1f-b12cec23f67d 2024-11-08T00:14:33,139 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/.tmp/hbase.id 2024-11-08T00:14:33,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741826_1002 (size=42) 2024-11-08T00:14:33,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741826_1002 (size=42) 2024-11-08T00:14:33,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741826_1002 (size=42) 2024-11-08T00:14:33,162 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/.tmp/hbase.id]:[hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/hbase.id] 2024-11-08T00:14:33,221 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:14:33,228 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T00:14:33,250 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-08T00:14:33,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741827_1003 (size=196) 2024-11-08T00:14:33,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741827_1003 (size=196) 2024-11-08T00:14:33,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741827_1003 (size=196) 2024-11-08T00:14:33,310 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:14:33,312 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T00:14:33,329 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:14:33,333 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:14:33,358 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:33,359 WARN [IPC Server handler 1 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:33,359 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:33,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741828_1004 (size=1189) 2024-11-08T00:14:33,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741828_1004 (size=1189) 2024-11-08T00:14:33,386 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store 2024-11-08T00:14:33,399 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:33,400 WARN [IPC Server handler 4 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:33,400 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:33,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741829_1005 (size=34) 2024-11-08T00:14:33,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741829_1005 (size=34) 2024-11-08T00:14:33,412 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T00:14:33,415 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:33,417 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T00:14:33,417 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:14:33,418 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:14:33,420 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T00:14:33,420 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:14:33,421 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:14:33,423 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731024873417Disabling compacts and flushes for region at 1731024873417Disabling writes for close at 1731024873420 (+3 ms)Writing region close event to WAL at 1731024873421 (+1 ms)Closed at 1731024873421 2024-11-08T00:14:33,426 WARN [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/.initializing 2024-11-08T00:14:33,426 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/WALs/3302f0f507bd,36429,1731024871043 2024-11-08T00:14:33,434 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:14:33,449 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3302f0f507bd%2C36429%2C1731024871043, suffix=, logDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/WALs/3302f0f507bd,36429,1731024871043, archiveDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/oldWALs, maxLogs=10 2024-11-08T00:14:33,482 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/WALs/3302f0f507bd,36429,1731024871043/3302f0f507bd%2C36429%2C1731024871043.1731024873455, exclude list is [], retry=0 2024-11-08T00:14:33,487 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:33,487 WARN [IPC Server handler 3 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:33,488 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:33,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44189,DS-697ab831-8e11-49a9-ae3f-5d4889ea6c9b,DISK] 2024-11-08T00:14:33,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36391,DS-c7c86592-a2a6-4354-ab71-9748dcc4c027,DISK] 2024-11-08T00:14:33,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T00:14:33,525 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:33,526 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:33,526 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:33,526 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:33,526 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:33,527 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:33,527 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:33,527 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:33,555 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/WALs/3302f0f507bd,36429,1731024871043/3302f0f507bd%2C36429%2C1731024871043.1731024873455 2024-11-08T00:14:33,556 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43187:43187),(127.0.0.1/127.0.0.1:44713:44713)] 2024-11-08T00:14:33,557 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:14:33,557 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:33,560 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,561 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T00:14:33,628 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:33,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:33,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,635 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T00:14:33,635 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:33,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:33,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T00:14:33,640 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:33,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:33,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T00:14:33,644 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:33,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:33,645 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,650 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,652 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,659 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,660 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,664 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:14:33,668 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:14:33,673 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:33,674 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68055221, jitterRate=0.014101818203926086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:14:33,682 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731024873575Initializing all the Stores at 1731024873578 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024873578Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024873579 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024873579Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024873579Cleaning up temporary data from old regions at 1731024873660 (+81 ms)Region opened successfully at 1731024873682 (+22 ms) 2024-11-08T00:14:33,683 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T00:14:33,715 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@861597d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3302f0f507bd/172.17.0.3:0 2024-11-08T00:14:33,744 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T00:14:33,754 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:33,754 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:33,757 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:33,758 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T00:14:33,763 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T00:14:33,763 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T00:14:33,788 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T00:14:33,796 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T00:14:33,804 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T00:14:33,806 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T00:14:33,807 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T00:14:33,814 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T00:14:33,817 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T00:14:33,821 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T00:14:33,833 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T00:14:33,836 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T00:14:33,928 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T00:14:33,946 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T00:14:33,951 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T00:14:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:14:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:14:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:14:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:14:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,965 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3302f0f507bd,36429,1731024871043, sessionid=0x10117cb57f40000, setting cluster-up flag (Was=false) 2024-11-08T00:14:33,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:33,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:34,025 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T00:14:34,028 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3302f0f507bd,36429,1731024871043 2024-11-08T00:14:34,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:34,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:34,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:34,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:34,088 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T00:14:34,090 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3302f0f507bd,36429,1731024871043 2024-11-08T00:14:34,097 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T00:14:34,170 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:34,180 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T00:14:34,183 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer(746): ClusterId : 2dc56221-06a4-403e-ac1f-b12cec23f67d 2024-11-08T00:14:34,183 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(746): ClusterId : 2dc56221-06a4-403e-ac1f-b12cec23f67d 2024-11-08T00:14:34,184 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(746): ClusterId : 2dc56221-06a4-403e-ac1f-b12cec23f67d 2024-11-08T00:14:34,186 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T00:14:34,186 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T00:14:34,186 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T00:14:34,187 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T00:14:34,193 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3302f0f507bd,36429,1731024871043 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T00:14:34,205 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T00:14:34,205 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T00:14:34,205 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T00:14:34,206 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T00:14:34,206 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T00:14:34,206 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T00:14:34,206 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3302f0f507bd:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:14:34,207 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3302f0f507bd:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:14:34,207 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3302f0f507bd:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:14:34,207 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3302f0f507bd:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:14:34,207 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3302f0f507bd:0, corePoolSize=10, maxPoolSize=10 2024-11-08T00:14:34,207 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,207 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3302f0f507bd:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:14:34,207 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,208 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024904208 2024-11-08T00:14:34,210 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T00:14:34,211 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T00:14:34,213 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:34,214 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T00:14:34,215 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T00:14:34,215 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T00:14:34,216 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T00:14:34,216 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T00:14:34,217 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,220 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T00:14:34,221 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T00:14:34,221 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T00:14:34,222 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:34,223 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T00:14:34,223 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T00:14:34,223 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T00:14:34,226 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.large.0-1731024874225,5,FailOnTimeoutGroup] 2024-11-08T00:14:34,226 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T00:14:34,227 DEBUG [RS:0;3302f0f507bd:38371 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be7c086, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3302f0f507bd/172.17.0.3:0 2024-11-08T00:14:34,227 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T00:14:34,227 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.small.0-1731024874226,5,FailOnTimeoutGroup] 2024-11-08T00:14:34,227 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,227 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T00:14:34,227 DEBUG [RS:2;3302f0f507bd:36139 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f1d145f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3302f0f507bd/172.17.0.3:0 2024-11-08T00:14:34,228 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,229 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T00:14:34,229 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,229 DEBUG [RS:1;3302f0f507bd:44033 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679d01e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3302f0f507bd/172.17.0.3:0 2024-11-08T00:14:34,249 DEBUG [RS:2;3302f0f507bd:36139 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;3302f0f507bd:36139 2024-11-08T00:14:34,250 DEBUG [RS:0;3302f0f507bd:38371 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3302f0f507bd:38371 2024-11-08T00:14:34,250 DEBUG [RS:1;3302f0f507bd:44033 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3302f0f507bd:44033 2024-11-08T00:14:34,253 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T00:14:34,253 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T00:14:34,253 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T00:14:34,253 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T00:14:34,253 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T00:14:34,253 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T00:14:34,253 DEBUG [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T00:14:34,253 DEBUG [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T00:14:34,253 DEBUG [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T00:14:34,256 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(2659): reportForDuty to master=3302f0f507bd,36429,1731024871043 with port=38371, startcode=1731024872767 2024-11-08T00:14:34,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741831_1007 (size=1321) 2024-11-08T00:14:34,256 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer(2659): reportForDuty to master=3302f0f507bd,36429,1731024871043 with port=44033, startcode=1731024872883 2024-11-08T00:14:34,256 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(2659): reportForDuty to master=3302f0f507bd,36429,1731024871043 with port=36139, startcode=1731024872925 2024-11-08T00:14:34,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741831_1007 (size=1321) 2024-11-08T00:14:34,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741831_1007 (size=1321) 2024-11-08T00:14:34,263 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T00:14:34,263 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:34,271 DEBUG [RS:1;3302f0f507bd:44033 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T00:14:34,271 DEBUG [RS:2;3302f0f507bd:36139 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T00:14:34,271 DEBUG [RS:0;3302f0f507bd:38371 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T00:14:34,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741832_1008 (size=32) 2024-11-08T00:14:34,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741832_1008 (size=32) 2024-11-08T00:14:34,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741832_1008 (size=32) 2024-11-08T00:14:34,324 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:34,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T00:14:34,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T00:14:34,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:34,340 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52363, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T00:14:34,340 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58849, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T00:14:34,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:34,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T00:14:34,344 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41277, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T00:14:34,345 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T00:14:34,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:34,348 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3302f0f507bd,44033,1731024872883 2024-11-08T00:14:34,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:34,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T00:14:34,351 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] master.ServerManager(517): Registering regionserver=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:34,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T00:14:34,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:34,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:34,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T00:14:34,359 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T00:14:34,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:34,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:34,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T00:14:34,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740 2024-11-08T00:14:34,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740 2024-11-08T00:14:34,366 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3302f0f507bd,38371,1731024872767 2024-11-08T00:14:34,367 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] master.ServerManager(517): Registering regionserver=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:34,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T00:14:34,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T00:14:34,371 DEBUG [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:34,371 DEBUG [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44875 2024-11-08T00:14:34,372 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:14:34,374 DEBUG [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:34,374 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3302f0f507bd,36139,1731024872925 2024-11-08T00:14:34,374 DEBUG [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44875 2024-11-08T00:14:34,374 DEBUG [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T00:14:34,374 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] master.ServerManager(517): Registering regionserver=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:34,374 DEBUG [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T00:14:34,380 DEBUG [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:34,380 DEBUG [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44875 2024-11-08T00:14:34,380 DEBUG [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T00:14:34,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T00:14:34,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:14:34,386 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:34,387 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75219872, jitterRate=0.12086343765258789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:14:34,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731024874324Initializing all the Stores at 1731024874333 (+9 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024874333Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024874333Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024874333Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024874333Cleaning up temporary data from old regions at 1731024874370 (+37 ms)Region opened successfully at 1731024874391 (+21 ms) 2024-11-08T00:14:34,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T00:14:34,391 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T00:14:34,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T00:14:34,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T00:14:34,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T00:14:34,393 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T00:14:34,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731024874391Disabling compacts and flushes for region at 1731024874391Disabling writes for close at 1731024874392 (+1 ms)Writing region close event to WAL at 1731024874393 (+1 ms)Closed at 1731024874393 2024-11-08T00:14:34,396 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:34,396 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T00:14:34,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T00:14:34,412 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:34,415 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T00:14:34,427 DEBUG [RS:0;3302f0f507bd:38371 {}] zookeeper.ZKUtil(111): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3302f0f507bd,38371,1731024872767 2024-11-08T00:14:34,427 WARN [RS:0;3302f0f507bd:38371 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:14:34,427 DEBUG [RS:1;3302f0f507bd:44033 {}] zookeeper.ZKUtil(111): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3302f0f507bd,44033,1731024872883 2024-11-08T00:14:34,427 INFO [RS:0;3302f0f507bd:38371 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:14:34,427 DEBUG [RS:2;3302f0f507bd:36139 {}] zookeeper.ZKUtil(111): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3302f0f507bd,36139,1731024872925 2024-11-08T00:14:34,427 WARN [RS:1;3302f0f507bd:44033 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:14:34,427 WARN [RS:2;3302f0f507bd:36139 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:14:34,427 INFO [RS:1;3302f0f507bd:44033 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:14:34,427 DEBUG [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,38371,1731024872767 2024-11-08T00:14:34,427 INFO [RS:2;3302f0f507bd:36139 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:14:34,427 DEBUG [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883 2024-11-08T00:14:34,428 DEBUG [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,36139,1731024872925 2024-11-08T00:14:34,428 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3302f0f507bd,38371,1731024872767] 2024-11-08T00:14:34,429 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3302f0f507bd,44033,1731024872883] 2024-11-08T00:14:34,429 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3302f0f507bd,36139,1731024872925] 2024-11-08T00:14:34,455 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T00:14:34,455 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T00:14:34,455 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T00:14:34,470 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T00:14:34,470 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T00:14:34,470 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T00:14:34,475 INFO [RS:0;3302f0f507bd:38371 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T00:14:34,475 INFO [RS:2;3302f0f507bd:36139 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T00:14:34,475 INFO [RS:1;3302f0f507bd:44033 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T00:14:34,475 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,475 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,475 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,476 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T00:14:34,476 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T00:14:34,476 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T00:14:34,482 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T00:14:34,482 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T00:14:34,482 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T00:14:34,483 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,483 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,483 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,483 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,483 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,483 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:14:34,484 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:14:34,484 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:14:34,484 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,484 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3302f0f507bd:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:14:34,485 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3302f0f507bd:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:14:34,485 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:0;3302f0f507bd:38371 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:14:34,485 DEBUG [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:14:34,485 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:14:34,485 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3302f0f507bd:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:14:34,485 DEBUG [RS:1;3302f0f507bd:44033 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:14:34,487 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,487 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,488 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,488 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,488 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,488 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,44033,1731024872883-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,38371,1731024872767-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:14:34,490 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36139,1731024872925-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:14:34,510 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T00:14:34,510 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T00:14:34,511 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T00:14:34,512 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,38371,1731024872767-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,512 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36139,1731024872925-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,512 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,44033,1731024872883-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,512 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,512 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,512 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.Replication(171): 3302f0f507bd,36139,1731024872925 started 2024-11-08T00:14:34,512 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,513 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.Replication(171): 3302f0f507bd,38371,1731024872767 started 2024-11-08T00:14:34,513 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.Replication(171): 3302f0f507bd,44033,1731024872883 started 2024-11-08T00:14:34,535 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,536 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(1482): Serving as 3302f0f507bd,38371,1731024872767, RpcServer on 3302f0f507bd/172.17.0.3:38371, sessionid=0x10117cb57f40001 2024-11-08T00:14:34,536 INFO [RS:1;3302f0f507bd:44033 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,537 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T00:14:34,537 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:34,537 DEBUG [RS:0;3302f0f507bd:38371 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3302f0f507bd,38371,1731024872767 2024-11-08T00:14:34,537 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer(1482): Serving as 3302f0f507bd,44033,1731024872883, RpcServer on 3302f0f507bd/172.17.0.3:44033, sessionid=0x10117cb57f40002 2024-11-08T00:14:34,537 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(1482): Serving as 3302f0f507bd,36139,1731024872925, RpcServer on 3302f0f507bd/172.17.0.3:36139, sessionid=0x10117cb57f40003 2024-11-08T00:14:34,537 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3302f0f507bd,38371,1731024872767' 2024-11-08T00:14:34,537 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T00:14:34,537 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T00:14:34,537 DEBUG [RS:1;3302f0f507bd:44033 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3302f0f507bd,44033,1731024872883 2024-11-08T00:14:34,537 DEBUG [RS:2;3302f0f507bd:36139 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3302f0f507bd,36139,1731024872925 2024-11-08T00:14:34,537 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3302f0f507bd,44033,1731024872883' 2024-11-08T00:14:34,537 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T00:14:34,537 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3302f0f507bd,36139,1731024872925' 2024-11-08T00:14:34,537 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T00:14:34,537 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T00:14:34,538 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T00:14:34,538 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T00:14:34,538 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T00:14:34,539 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T00:14:34,539 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T00:14:34,539 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T00:14:34,539 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T00:14:34,539 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T00:14:34,539 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T00:14:34,540 DEBUG [RS:0;3302f0f507bd:38371 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3302f0f507bd,38371,1731024872767 2024-11-08T00:14:34,540 DEBUG [RS:1;3302f0f507bd:44033 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3302f0f507bd,44033,1731024872883 2024-11-08T00:14:34,540 DEBUG [RS:2;3302f0f507bd:36139 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3302f0f507bd,36139,1731024872925 2024-11-08T00:14:34,540 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3302f0f507bd,36139,1731024872925' 2024-11-08T00:14:34,540 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3302f0f507bd,38371,1731024872767' 2024-11-08T00:14:34,540 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3302f0f507bd,44033,1731024872883' 2024-11-08T00:14:34,540 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T00:14:34,540 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T00:14:34,540 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T00:14:34,540 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T00:14:34,541 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T00:14:34,541 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T00:14:34,541 DEBUG [RS:2;3302f0f507bd:36139 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T00:14:34,541 INFO [RS:2;3302f0f507bd:36139 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T00:14:34,541 INFO [RS:2;3302f0f507bd:36139 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T00:14:34,541 DEBUG [RS:0;3302f0f507bd:38371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T00:14:34,541 INFO [RS:0;3302f0f507bd:38371 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T00:14:34,541 DEBUG [RS:1;3302f0f507bd:44033 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T00:14:34,541 INFO [RS:0;3302f0f507bd:38371 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T00:14:34,541 INFO [RS:1;3302f0f507bd:44033 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T00:14:34,541 INFO [RS:1;3302f0f507bd:44033 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T00:14:34,566 WARN [3302f0f507bd:36429 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-08T00:14:34,646 INFO [RS:2;3302f0f507bd:36139 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:14:34,646 INFO [RS:0;3302f0f507bd:38371 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:14:34,646 INFO [RS:1;3302f0f507bd:44033 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:14:34,649 INFO [RS:0;3302f0f507bd:38371 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3302f0f507bd%2C38371%2C1731024872767, suffix=, logDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,38371,1731024872767, archiveDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs, maxLogs=32 2024-11-08T00:14:34,649 INFO [RS:1;3302f0f507bd:44033 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3302f0f507bd%2C44033%2C1731024872883, suffix=, logDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883, archiveDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs, maxLogs=32 2024-11-08T00:14:34,649 INFO [RS:2;3302f0f507bd:36139 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3302f0f507bd%2C36139%2C1731024872925, suffix=, logDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,36139,1731024872925, archiveDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs, maxLogs=32 2024-11-08T00:14:34,665 DEBUG [RS:1;3302f0f507bd:44033 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883/3302f0f507bd%2C44033%2C1731024872883.1731024874653, exclude list is [], retry=0 2024-11-08T00:14:34,665 DEBUG [RS:0;3302f0f507bd:38371 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,38371,1731024872767/3302f0f507bd%2C38371%2C1731024872767.1731024874653, exclude list is [], retry=0 2024-11-08T00:14:34,665 DEBUG [RS:2;3302f0f507bd:36139 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,36139,1731024872925/3302f0f507bd%2C36139%2C1731024872925.1731024874653, exclude list is [], retry=0 2024-11-08T00:14:34,668 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:34,668 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:34,668 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:34,668 WARN [IPC Server handler 4 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:34,668 WARN [IPC Server handler 3 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:34,668 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:34,668 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:34,668 WARN [IPC Server handler 1 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:34,669 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:34,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36391,DS-c7c86592-a2a6-4354-ab71-9748dcc4c027,DISK] 2024-11-08T00:14:34,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44189,DS-697ab831-8e11-49a9-ae3f-5d4889ea6c9b,DISK] 2024-11-08T00:14:34,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36391,DS-c7c86592-a2a6-4354-ab71-9748dcc4c027,DISK] 2024-11-08T00:14:34,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44189,DS-697ab831-8e11-49a9-ae3f-5d4889ea6c9b,DISK] 2024-11-08T00:14:34,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36391,DS-c7c86592-a2a6-4354-ab71-9748dcc4c027,DISK] 2024-11-08T00:14:34,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44189,DS-697ab831-8e11-49a9-ae3f-5d4889ea6c9b,DISK] 2024-11-08T00:14:34,726 INFO [RS:0;3302f0f507bd:38371 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,38371,1731024872767/3302f0f507bd%2C38371%2C1731024872767.1731024874653 2024-11-08T00:14:34,729 DEBUG [RS:0;3302f0f507bd:38371 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44713:44713),(127.0.0.1/127.0.0.1:43187:43187)] 2024-11-08T00:14:34,729 INFO [RS:1;3302f0f507bd:44033 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883/3302f0f507bd%2C44033%2C1731024872883.1731024874653 2024-11-08T00:14:34,729 INFO [RS:2;3302f0f507bd:36139 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,36139,1731024872925/3302f0f507bd%2C36139%2C1731024872925.1731024874653 2024-11-08T00:14:34,734 DEBUG [RS:2;3302f0f507bd:36139 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43187:43187),(127.0.0.1/127.0.0.1:44713:44713)] 2024-11-08T00:14:34,734 DEBUG [RS:1;3302f0f507bd:44033 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44713:44713),(127.0.0.1/127.0.0.1:43187:43187)] 2024-11-08T00:14:34,819 DEBUG [3302f0f507bd:36429 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-08T00:14:34,828 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(204): Hosts are {3302f0f507bd=0} racks are {/default-rack=0} 2024-11-08T00:14:34,834 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T00:14:34,834 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T00:14:34,834 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T00:14:34,834 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T00:14:34,834 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T00:14:34,834 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T00:14:34,834 INFO [3302f0f507bd:36429 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T00:14:34,834 INFO [3302f0f507bd:36429 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T00:14:34,835 INFO [3302f0f507bd:36429 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T00:14:34,835 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T00:14:34,842 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:34,850 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,44033,1731024872883, state=OPENING 2024-11-08T00:14:34,864 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T00:14:34,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:34,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:34,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:34,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:34,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:34,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:34,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:34,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:34,883 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:34,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,44033,1731024872883}] 2024-11-08T00:14:35,061 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T00:14:35,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45947, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T00:14:35,075 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T00:14:35,076 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:14:35,076 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T00:14:35,079 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3302f0f507bd%2C44033%2C1731024872883.meta, suffix=.meta, logDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883, archiveDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs, maxLogs=32 2024-11-08T00:14:35,094 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, exclude list is [], retry=0 2024-11-08T00:14:35,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36957,DS-05680217-d0db-4d7d-a701-2d3460612161,DISK] 2024-11-08T00:14:35,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44189,DS-697ab831-8e11-49a9-ae3f-5d4889ea6c9b,DISK] 2024-11-08T00:14:35,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36391,DS-c7c86592-a2a6-4354-ab71-9748dcc4c027,DISK] 2024-11-08T00:14:35,102 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta 2024-11-08T00:14:35,102 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39279:39279),(127.0.0.1/127.0.0.1:44713:44713),(127.0.0.1/127.0.0.1:43187:43187)] 2024-11-08T00:14:35,103 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:14:35,104 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T00:14:35,107 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T00:14:35,111 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T00:14:35,115 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T00:14:35,115 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,116 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T00:14:35,116 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T00:14:35,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T00:14:35,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T00:14:35,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:35,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:35,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T00:14:35,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T00:14:35,123 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:35,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:35,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T00:14:35,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T00:14:35,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:35,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:35,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T00:14:35,127 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T00:14:35,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:35,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:35,128 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T00:14:35,130 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740 2024-11-08T00:14:35,132 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740 2024-11-08T00:14:35,135 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T00:14:35,135 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T00:14:35,136 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:14:35,138 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T00:14:35,140 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64777578, jitterRate=-0.03473886847496033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:14:35,140 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T00:14:35,141 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731024875116Writing region info on filesystem at 1731024875116Initializing all the Stores at 1731024875118 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024875118Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024875118Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024875118Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024875119 (+1 ms)Cleaning up temporary data from old regions at 1731024875135 (+16 ms)Running coprocessor post-open hooks at 1731024875140 (+5 ms)Region opened successfully at 1731024875141 (+1 ms) 2024-11-08T00:14:35,147 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731024875052 2024-11-08T00:14:35,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:35,161 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T00:14:35,162 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T00:14:35,164 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,44033,1731024872883, state=OPEN 2024-11-08T00:14:35,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:35,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:35,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:35,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:35,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:35,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:35,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:35,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:35,173 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:35,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T00:14:35,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,44033,1731024872883 in 288 msec 2024-11-08T00:14:35,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T00:14:35,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 779 msec 2024-11-08T00:14:35,188 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:35,188 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T00:14:35,206 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:14:35,207 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1] 2024-11-08T00:14:35,230 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:14:35,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45571, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:14:35,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1230 sec 2024-11-08T00:14:35,253 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731024875253, completionTime=-1 2024-11-08T00:14:35,256 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-08T00:14:35,257 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:35,283 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-08T00:14:35,283 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024935283 2024-11-08T00:14:35,283 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731024995283 2024-11-08T00:14:35,283 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 26 msec 2024-11-08T00:14:35,285 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:35,291 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36429,1731024871043-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:35,291 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36429,1731024871043-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:35,291 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36429,1731024871043-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:35,293 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3302f0f507bd:36429, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:35,293 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:35,294 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:35,301 DEBUG [master/3302f0f507bd:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T00:14:35,321 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.243sec 2024-11-08T00:14:35,323 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T00:14:35,324 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T00:14:35,325 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T00:14:35,325 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T00:14:35,325 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T00:14:35,326 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36429,1731024871043-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:14:35,327 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36429,1731024871043-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T00:14:35,332 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T00:14:35,333 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T00:14:35,334 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36429,1731024871043-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:14:35,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b48c89d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:14:35,397 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T00:14:35,397 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T00:14:35,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36429,-1 for getting cluster id 2024-11-08T00:14:35,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:14:35,409 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2dc56221-06a4-403e-ac1f-b12cec23f67d' 2024-11-08T00:14:35,411 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:14:35,412 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2dc56221-06a4-403e-ac1f-b12cec23f67d" 2024-11-08T00:14:35,412 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a605a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:14:35,412 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36429,-1] 2024-11-08T00:14:35,416 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:14:35,418 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:35,419 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:14:35,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d42a85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:14:35,422 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:14:35,429 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1] 2024-11-08T00:14:35,430 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:14:35,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:14:35,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3302f0f507bd,36429,1731024871043 2024-11-08T00:14:35,455 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:14:35,460 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3302f0f507bd,36429,1731024871043 2024-11-08T00:14:35,463 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@15e92a53 2024-11-08T00:14:35,463 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:14:35,466 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:14:35,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:14:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:35,509 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T00:14:35,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-08T00:14:35,512 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:35,515 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T00:14:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:14:35,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741837_1013 (size=358) 2024-11-08T00:14:35,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741837_1013 (size=358) 2024-11-08T00:14:35,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741837_1013 (size=358) 2024-11-08T00:14:35,544 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8624a8443e655fa753c4de0934012d54, NAME => 'test,,1731024875468.8624a8443e655fa753c4de0934012d54.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,545 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9c5dafc4711af09b862954533a7784ca, NAME => 'test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,545 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a594c54df520141421a62d3f056e15f2, NAME => 'test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,550 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,550 WARN [IPC Server handler 1 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,550 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,553 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,553 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,553 WARN [IPC Server handler 4 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,553 WARN [IPC Server handler 0 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,553 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,553 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741838_1014 (size=42) 2024-11-08T00:14:35,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741838_1014 (size=42) 2024-11-08T00:14:35,567 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731024875468.8624a8443e655fa753c4de0934012d54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8624a8443e655fa753c4de0934012d54, disabling compactions & flushes 2024-11-08T00:14:35,568 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:35,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:35,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731024875468.8624a8443e655fa753c4de0934012d54. after waiting 0 ms 2024-11-08T00:14:35,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:35,568 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:35,568 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8624a8443e655fa753c4de0934012d54: Waiting for close lock at 1731024875567Disabling compacts and flushes for region at 1731024875567Disabling writes for close at 1731024875568 (+1 ms)Writing region close event to WAL at 1731024875568Closed at 1731024875568 2024-11-08T00:14:35,568 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ffaf29acbc927f398077e41ad232c549, NAME => 'test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741839_1015 (size=45) 2024-11-08T00:14:35,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741839_1015 (size=45) 2024-11-08T00:14:35,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9c5dafc4711af09b862954533a7784ca, disabling compactions & flushes 2024-11-08T00:14:35,577 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:35,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:35,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. after waiting 0 ms 2024-11-08T00:14:35,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:35,577 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:35,577 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9c5dafc4711af09b862954533a7784ca: Waiting for close lock at 1731024875577Disabling compacts and flushes for region at 1731024875577Disabling writes for close at 1731024875577Writing region close event to WAL at 1731024875577Closed at 1731024875577 2024-11-08T00:14:35,578 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cd2627434f69d79290b6a35838b5f326, NAME => 'test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741840_1016 (size=45) 2024-11-08T00:14:35,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741840_1016 (size=45) 2024-11-08T00:14:35,585 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,586 WARN [IPC Server handler 3 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,586 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a594c54df520141421a62d3f056e15f2, disabling compactions & flushes 2024-11-08T00:14:35,587 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:35,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:35,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. after waiting 0 ms 2024-11-08T00:14:35,587 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:35,588 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:35,588 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a594c54df520141421a62d3f056e15f2: Waiting for close lock at 1731024875587Disabling compacts and flushes for region at 1731024875587Disabling writes for close at 1731024875587Writing region close event to WAL at 1731024875587Closed at 1731024875587 2024-11-08T00:14:35,589 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 66bf32aac498176881371b724c9d8008, NAME => 'test,fff,1731024875468.66bf32aac498176881371b724c9d8008.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,589 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,590 WARN [IPC Server handler 1 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,590 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,597 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,598 WARN [IPC Server handler 3 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,598 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741841_1017 (size=45) 2024-11-08T00:14:35,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741841_1017 (size=45) 2024-11-08T00:14:35,605 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,605 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ffaf29acbc927f398077e41ad232c549, disabling compactions & flushes 2024-11-08T00:14:35,605 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:35,605 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:35,605 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. after waiting 0 ms 2024-11-08T00:14:35,605 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:35,605 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:35,605 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ffaf29acbc927f398077e41ad232c549: Waiting for close lock at 1731024875605Disabling compacts and flushes for region at 1731024875605Disabling writes for close at 1731024875605Writing region close event to WAL at 1731024875605Closed at 1731024875605 2024-11-08T00:14:35,607 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => faad0b2f1a2327cef2cd0aefa9a4d1c8, NAME => 'test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741842_1018 (size=45) 2024-11-08T00:14:35,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741842_1018 (size=45) 2024-11-08T00:14:35,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741843_1019 (size=45) 2024-11-08T00:14:35,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741843_1019 (size=45) 2024-11-08T00:14:35,612 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1731024875468.66bf32aac498176881371b724c9d8008.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,612 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 66bf32aac498176881371b724c9d8008, disabling compactions & flushes 2024-11-08T00:14:35,612 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:35,612 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:35,612 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731024875468.66bf32aac498176881371b724c9d8008. after waiting 0 ms 2024-11-08T00:14:35,612 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:35,612 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:35,612 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 66bf32aac498176881371b724c9d8008: Waiting for close lock at 1731024875612Disabling compacts and flushes for region at 1731024875612Disabling writes for close at 1731024875612Writing region close event to WAL at 1731024875612Closed at 1731024875612 2024-11-08T00:14:35,613 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b5026dfa3bad2942533f117d619b1656, NAME => 'test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741844_1020 (size=45) 2024-11-08T00:14:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741844_1020 (size=45) 2024-11-08T00:14:35,621 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741844_1020 (size=45) 2024-11-08T00:14:35,621 WARN [IPC Server handler 4 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,621 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,622 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing faad0b2f1a2327cef2cd0aefa9a4d1c8, disabling compactions & flushes 2024-11-08T00:14:35,622 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8. 2024-11-08T00:14:35,623 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8. 2024-11-08T00:14:35,623 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8. after waiting 0 ms 2024-11-08T00:14:35,623 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8. 2024-11-08T00:14:35,623 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8. 2024-11-08T00:14:35,623 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for faad0b2f1a2327cef2cd0aefa9a4d1c8: Waiting for close lock at 1731024875622Disabling compacts and flushes for region at 1731024875622Disabling writes for close at 1731024875623 (+1 ms)Writing region close event to WAL at 1731024875623Closed at 1731024875623 2024-11-08T00:14:35,624 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 13f20a01a04306fecc9b545706c9d14c, NAME => 'test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:14:35,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741845_1021 (size=45) 2024-11-08T00:14:35,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741845_1021 (size=45) 2024-11-08T00:14:35,628 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,629 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b5026dfa3bad2942533f117d619b1656, disabling compactions & flushes 2024-11-08T00:14:35,629 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:35,629 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:35,629 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,629 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. after waiting 0 ms 2024-11-08T00:14:35,629 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:35,629 WARN [IPC Server handler 3 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,629 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:35,629 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,629 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b5026dfa3bad2942533f117d619b1656: Waiting for close lock at 1731024875629Disabling compacts and flushes for region at 1731024875629Disabling writes for close at 1731024875629Writing region close event to WAL at 1731024875629Closed at 1731024875629 2024-11-08T00:14:35,629 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2ef74f869fdb87a20df612d1e6e2c262, NAME => 'test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,633 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,633 WARN [IPC Server handler 0 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,633 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741846_1022 (size=45) 2024-11-08T00:14:35,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741846_1022 (size=45) 2024-11-08T00:14:35,643 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 13f20a01a04306fecc9b545706c9d14c, disabling compactions & flushes 2024-11-08T00:14:35,644 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:35,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:35,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. after waiting 0 ms 2024-11-08T00:14:35,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:35,644 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:35,644 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 13f20a01a04306fecc9b545706c9d14c: Waiting for close lock at 1731024875644Disabling compacts and flushes for region at 1731024875644Disabling writes for close at 1731024875644Writing region close event to WAL at 1731024875644Closed at 1731024875644 2024-11-08T00:14:35,645 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f04142b3a962ffb3cc6e9629a0aa3c6a, NAME => 'test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741847_1023 (size=45) 2024-11-08T00:14:35,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741847_1023 (size=45) 2024-11-08T00:14:35,647 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,647 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2ef74f869fdb87a20df612d1e6e2c262, disabling compactions & flushes 2024-11-08T00:14:35,647 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:35,647 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:35,647 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. after waiting 0 ms 2024-11-08T00:14:35,647 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:35,648 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:35,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2ef74f869fdb87a20df612d1e6e2c262: Waiting for close lock at 1731024875647Disabling compacts and flushes for region at 1731024875647Disabling writes for close at 1731024875647Writing region close event to WAL at 1731024875648 (+1 ms)Closed at 1731024875648 2024-11-08T00:14:35,649 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3bb82385d0b72eb69ca4bdc52663bf8a, NAME => 'test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,649 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,650 WARN [IPC Server handler 0 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,650 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,654 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,655 WARN [IPC Server handler 4 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,655 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741848_1024 (size=45) 2024-11-08T00:14:35,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741848_1024 (size=45) 2024-11-08T00:14:35,657 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,657 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f04142b3a962ffb3cc6e9629a0aa3c6a, disabling compactions & flushes 2024-11-08T00:14:35,657 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:35,657 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:35,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. after waiting 0 ms 2024-11-08T00:14:35,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:35,658 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:35,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f04142b3a962ffb3cc6e9629a0aa3c6a: Waiting for close lock at 1731024875657Disabling compacts and flushes for region at 1731024875657Disabling writes for close at 1731024875658 (+1 ms)Writing region close event to WAL at 1731024875658Closed at 1731024875658 2024-11-08T00:14:35,658 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1c8336e737af786ef3c90e95f98c755f, NAME => 'test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,663 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,663 WARN [IPC Server handler 1 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,663 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741849_1025 (size=45) 2024-11-08T00:14:35,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741849_1025 (size=45) 2024-11-08T00:14:35,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741850_1026 (size=45) 2024-11-08T00:14:35,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741850_1026 (size=45) 2024-11-08T00:14:35,671 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,672 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1c8336e737af786ef3c90e95f98c755f, disabling compactions & flushes 2024-11-08T00:14:35,672 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:35,672 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:35,672 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. after waiting 0 ms 2024-11-08T00:14:35,672 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:35,672 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:35,672 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1c8336e737af786ef3c90e95f98c755f: Waiting for close lock at 1731024875672Disabling compacts and flushes for region at 1731024875672Disabling writes for close at 1731024875672Writing region close event to WAL at 1731024875672Closed at 1731024875672 2024-11-08T00:14:35,673 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee421baf774947d334ec46882ae42d9c, NAME => 'test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,677 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,677 WARN [IPC Server handler 3 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,677 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741851_1027 (size=45) 2024-11-08T00:14:35,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741851_1027 (size=45) 2024-11-08T00:14:35,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ee421baf774947d334ec46882ae42d9c, disabling compactions & flushes 2024-11-08T00:14:35,685 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c. 2024-11-08T00:14:35,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c. 2024-11-08T00:14:35,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c. after waiting 0 ms 2024-11-08T00:14:35,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c. 2024-11-08T00:14:35,685 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c. 2024-11-08T00:14:35,685 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee421baf774947d334ec46882ae42d9c: Waiting for close lock at 1731024875685Disabling compacts and flushes for region at 1731024875685Disabling writes for close at 1731024875685Writing region close event to WAL at 1731024875685Closed at 1731024875685 2024-11-08T00:14:35,686 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3bddcc13c4eccc1d32b3c09b1dcb1583, NAME => 'test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741852_1028 (size=45) 2024-11-08T00:14:35,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741852_1028 (size=45) 2024-11-08T00:14:35,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741852_1028 (size=45) 2024-11-08T00:14:35,702 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,702 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3bddcc13c4eccc1d32b3c09b1dcb1583, disabling compactions & flushes 2024-11-08T00:14:35,702 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:35,702 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:35,703 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. after waiting 0 ms 2024-11-08T00:14:35,703 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:35,703 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:35,703 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3bddcc13c4eccc1d32b3c09b1dcb1583: Waiting for close lock at 1731024875702Disabling compacts and flushes for region at 1731024875702Disabling writes for close at 1731024875703 (+1 ms)Writing region close event to WAL at 1731024875703Closed at 1731024875703 2024-11-08T00:14:35,703 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 850fbeeee9e2784d3676499ac7582114, NAME => 'test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,707 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,708 WARN [IPC Server handler 3 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,708 WARN [IPC Server handler 3 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741853_1029 (size=45) 2024-11-08T00:14:35,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741853_1029 (size=45) 2024-11-08T00:14:35,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 850fbeeee9e2784d3676499ac7582114, disabling compactions & flushes 2024-11-08T00:14:35,714 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:35,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:35,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. after waiting 0 ms 2024-11-08T00:14:35,714 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:35,714 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:35,715 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 850fbeeee9e2784d3676499ac7582114: Waiting for close lock at 1731024875714Disabling compacts and flushes for region at 1731024875714Disabling writes for close at 1731024875714Writing region close event to WAL at 1731024875714Closed at 1731024875714 2024-11-08T00:14:35,715 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7acb38c712f4cad61cf5f78f7ffa9a18, NAME => 'test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,718 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,718 WARN [IPC Server handler 4 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,719 WARN [IPC Server handler 4 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741854_1030 (size=45) 2024-11-08T00:14:35,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741854_1030 (size=45) 2024-11-08T00:14:35,724 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,725 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7acb38c712f4cad61cf5f78f7ffa9a18, disabling compactions & flushes 2024-11-08T00:14:35,725 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:35,725 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:35,725 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. after waiting 0 ms 2024-11-08T00:14:35,725 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:35,725 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:35,725 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7acb38c712f4cad61cf5f78f7ffa9a18: Waiting for close lock at 1731024875725Disabling compacts and flushes for region at 1731024875725Disabling writes for close at 1731024875725Writing region close event to WAL at 1731024875725Closed at 1731024875725 2024-11-08T00:14:35,726 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b8e8ac9592d4e973f9896a6807e10272, NAME => 'test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,729 WARN [IPC Server handler 2 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,729 WARN [IPC Server handler 2 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,729 WARN [IPC Server handler 2 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741855_1031 (size=45) 2024-11-08T00:14:35,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741855_1031 (size=45) 2024-11-08T00:14:35,736 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,736 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b8e8ac9592d4e973f9896a6807e10272, disabling compactions & flushes 2024-11-08T00:14:35,737 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:35,737 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:35,737 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. after waiting 0 ms 2024-11-08T00:14:35,737 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:35,737 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:35,737 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b8e8ac9592d4e973f9896a6807e10272: Waiting for close lock at 1731024875736Disabling compacts and flushes for region at 1731024875736Disabling writes for close at 1731024875737 (+1 ms)Writing region close event to WAL at 1731024875737Closed at 1731024875737 2024-11-08T00:14:35,738 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2e10c5ec7fdc210db94ac8df9187f1c4, NAME => 'test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,741 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,741 WARN [IPC Server handler 0 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,741 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741856_1032 (size=45) 2024-11-08T00:14:35,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741856_1032 (size=45) 2024-11-08T00:14:35,747 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,747 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2e10c5ec7fdc210db94ac8df9187f1c4, disabling compactions & flushes 2024-11-08T00:14:35,747 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:35,747 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:35,747 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. after waiting 0 ms 2024-11-08T00:14:35,747 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:35,747 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:35,747 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2e10c5ec7fdc210db94ac8df9187f1c4: Waiting for close lock at 1731024875747Disabling compacts and flushes for region at 1731024875747Disabling writes for close at 1731024875747Writing region close event to WAL at 1731024875747Closed at 1731024875747 2024-11-08T00:14:35,748 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8280a310195422b4e0184bc7c709a2e9, NAME => 'test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,752 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,752 WARN [IPC Server handler 1 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,752 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741857_1033 (size=45) 2024-11-08T00:14:35,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741857_1033 (size=45) 2024-11-08T00:14:35,758 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,758 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8280a310195422b4e0184bc7c709a2e9, disabling compactions & flushes 2024-11-08T00:14:35,758 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. 2024-11-08T00:14:35,758 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. 2024-11-08T00:14:35,758 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. after waiting 0 ms 2024-11-08T00:14:35,758 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. 2024-11-08T00:14:35,758 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. 2024-11-08T00:14:35,759 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8280a310195422b4e0184bc7c709a2e9: Waiting for close lock at 1731024875758Disabling compacts and flushes for region at 1731024875758Disabling writes for close at 1731024875758Writing region close event to WAL at 1731024875758Closed at 1731024875758 2024-11-08T00:14:35,759 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 54a0ce095f1e6524de8bdf904569509f, NAME => 'test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741858_1034 (size=45) 2024-11-08T00:14:35,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741858_1034 (size=45) 2024-11-08T00:14:35,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741858_1034 (size=45) 2024-11-08T00:14:35,771 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,771 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 54a0ce095f1e6524de8bdf904569509f, disabling compactions & flushes 2024-11-08T00:14:35,771 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:35,771 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:35,771 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. after waiting 0 ms 2024-11-08T00:14:35,772 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:35,772 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:35,772 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 54a0ce095f1e6524de8bdf904569509f: Waiting for close lock at 1731024875771Disabling compacts and flushes for region at 1731024875771Disabling writes for close at 1731024875771Writing region close event to WAL at 1731024875772 (+1 ms)Closed at 1731024875772 2024-11-08T00:14:35,772 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bcb90ad7cb983c915be269c865806709, NAME => 'test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741859_1035 (size=45) 2024-11-08T00:14:35,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741859_1035 (size=45) 2024-11-08T00:14:35,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741859_1035 (size=45) 2024-11-08T00:14:35,794 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,794 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bcb90ad7cb983c915be269c865806709, disabling compactions & flushes 2024-11-08T00:14:35,794 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:35,794 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:35,794 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. after waiting 0 ms 2024-11-08T00:14:35,794 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:35,794 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:35,795 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bcb90ad7cb983c915be269c865806709: Waiting for close lock at 1731024875794Disabling compacts and flushes for region at 1731024875794Disabling writes for close at 1731024875794Writing region close event to WAL at 1731024875794Closed at 1731024875794 2024-11-08T00:14:35,795 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 40e3fed5cf6cc7497a1c2220e1de9b9b, NAME => 'test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,800 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,800 WARN [IPC Server handler 0 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,800 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741860_1036 (size=45) 2024-11-08T00:14:35,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741860_1036 (size=45) 2024-11-08T00:14:35,809 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,809 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 40e3fed5cf6cc7497a1c2220e1de9b9b, disabling compactions & flushes 2024-11-08T00:14:35,809 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:35,809 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:35,809 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. after waiting 0 ms 2024-11-08T00:14:35,809 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:35,809 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:35,809 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 40e3fed5cf6cc7497a1c2220e1de9b9b: Waiting for close lock at 1731024875809Disabling compacts and flushes for region at 1731024875809Disabling writes for close at 1731024875809Writing region close event to WAL at 1731024875809Closed at 1731024875809 2024-11-08T00:14:35,810 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 335e583f3dd0eda08c4c42b80a743311, NAME => 'test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741861_1037 (size=45) 2024-11-08T00:14:35,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741861_1037 (size=45) 2024-11-08T00:14:35,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741861_1037 (size=45) 2024-11-08T00:14:35,824 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,824 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 335e583f3dd0eda08c4c42b80a743311, disabling compactions & flushes 2024-11-08T00:14:35,825 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:35,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:35,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. after waiting 0 ms 2024-11-08T00:14:35,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:35,825 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:35,825 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 335e583f3dd0eda08c4c42b80a743311: Waiting for close lock at 1731024875824Disabling compacts and flushes for region at 1731024875824Disabling writes for close at 1731024875825 (+1 ms)Writing region close event to WAL at 1731024875825Closed at 1731024875825 2024-11-08T00:14:35,826 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f09fb3c8c48c1b2f01441065e691680c, NAME => 'test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,830 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,830 WARN [IPC Server handler 0 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,830 WARN [IPC Server handler 0 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741862_1038 (size=45) 2024-11-08T00:14:35,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741862_1038 (size=45) 2024-11-08T00:14:35,835 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,836 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f09fb3c8c48c1b2f01441065e691680c, disabling compactions & flushes 2024-11-08T00:14:35,836 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:35,836 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:35,836 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. after waiting 0 ms 2024-11-08T00:14:35,836 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:35,836 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:35,836 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f09fb3c8c48c1b2f01441065e691680c: Waiting for close lock at 1731024875835Disabling compacts and flushes for region at 1731024875835Disabling writes for close at 1731024875836 (+1 ms)Writing region close event to WAL at 1731024875836Closed at 1731024875836 2024-11-08T00:14:35,837 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c816073c18c6a09db7883f3602eda012, NAME => 'test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20 2024-11-08T00:14:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:14:35,841 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T00:14:35,841 WARN [IPC Server handler 1 on default port 44875 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T00:14:35,841 WARN [IPC Server handler 1 on default port 44875 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T00:14:35,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741863_1039 (size=42) 2024-11-08T00:14:35,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741863_1039 (size=42) 2024-11-08T00:14:35,846 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:35,847 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c816073c18c6a09db7883f3602eda012, disabling compactions & flushes 2024-11-08T00:14:35,847 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012. 2024-11-08T00:14:35,847 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012. 2024-11-08T00:14:35,847 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012. after waiting 0 ms 2024-11-08T00:14:35,847 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012. 2024-11-08T00:14:35,847 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012. 2024-11-08T00:14:35,847 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c816073c18c6a09db7883f3602eda012: Waiting for close lock at 1731024875847Disabling compacts and flushes for region at 1731024875847Disabling writes for close at 1731024875847Writing region close event to WAL at 1731024875847Closed at 1731024875847 2024-11-08T00:14:36,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cd2627434f69d79290b6a35838b5f326, disabling compactions & flushes 2024-11-08T00:14:36,009 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:36,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:36,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. after waiting 0 ms 2024-11-08T00:14:36,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:36,009 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:36,009 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cd2627434f69d79290b6a35838b5f326: Waiting for close lock at 1731024876009Disabling compacts and flushes for region at 1731024876009Disabling writes for close at 1731024876009Writing region close event to WAL at 1731024876009Closed at 1731024876009 2024-11-08T00:14:36,065 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,065 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 3bb82385d0b72eb69ca4bdc52663bf8a, disabling compactions & flushes 2024-11-08T00:14:36,065 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:36,065 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:36,065 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. after waiting 0 ms 2024-11-08T00:14:36,065 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:36,065 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:36,066 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3bb82385d0b72eb69ca4bdc52663bf8a: Waiting for close lock at 1731024876065Disabling compacts and flushes for region at 1731024876065Disabling writes for close at 1731024876065Writing region close event to WAL at 1731024876065Closed at 1731024876065 2024-11-08T00:14:36,070 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T00:14:36,076 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731024875468.8624a8443e655fa753c4de0934012d54.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,076 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,076 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,077 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,077 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731024875468.66bf32aac498176881371b724c9d8008.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,077 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,077 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,078 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,078 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,078 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,078 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,078 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,079 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,079 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,079 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,079 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,080 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,080 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,080 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,080 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,080 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,081 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,081 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,081 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,081 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,081 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731024876070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731024876070"}]},"ts":"1731024876070"} 2024-11-08T00:14:36,123 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-08T00:14:36,125 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T00:14:36,128 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731024876125"}]},"ts":"1731024876125"} 2024-11-08T00:14:36,132 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-08T00:14:36,133 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {3302f0f507bd=0} racks are {/default-rack=0} 2024-11-08T00:14:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:14:36,149 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T00:14:36,149 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T00:14:36,149 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T00:14:36,149 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T00:14:36,149 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T00:14:36,149 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T00:14:36,149 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T00:14:36,149 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T00:14:36,149 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T00:14:36,149 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T00:14:36,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8624a8443e655fa753c4de0934012d54, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c5dafc4711af09b862954533a7784ca, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ffaf29acbc927f398077e41ad232c549, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd2627434f69d79290b6a35838b5f326, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66bf32aac498176881371b724c9d8008, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b5026dfa3bad2942533f117d619b1656, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=13f20a01a04306fecc9b545706c9d14c, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f04142b3a962ffb3cc6e9629a0aa3c6a, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3bb82385d0b72eb69ca4bdc52663bf8a, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1c8336e737af786ef3c90e95f98c755f, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3bddcc13c4eccc1d32b3c09b1dcb1583, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=850fbeeee9e2784d3676499ac7582114, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b8e8ac9592d4e973f9896a6807e10272, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e10c5ec7fdc210db94ac8df9187f1c4, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=54a0ce095f1e6524de8bdf904569509f, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bcb90ad7cb983c915be269c865806709, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=335e583f3dd0eda08c4c42b80a743311, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f09fb3c8c48c1b2f01441065e691680c, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN}] 2024-11-08T00:14:36,161 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:36,163 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN; state=OFFLINE, location=3302f0f507bd,44033,1731024872883; forceNewPlan=false, retain=false 2024-11-08T00:14:36,164 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f09fb3c8c48c1b2f01441065e691680c, ASSIGN 2024-11-08T00:14:36,166 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f09fb3c8c48c1b2f01441065e691680c, ASSIGN; state=OFFLINE, location=3302f0f507bd,38371,1731024872767; forceNewPlan=false, retain=false 2024-11-08T00:14:36,168 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=335e583f3dd0eda08c4c42b80a743311, ASSIGN 2024-11-08T00:14:36,169 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=335e583f3dd0eda08c4c42b80a743311, ASSIGN; state=OFFLINE, location=3302f0f507bd,36139,1731024872925; forceNewPlan=false, retain=false 2024-11-08T00:14:36,170 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:36,172 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN; state=OFFLINE, location=3302f0f507bd,44033,1731024872883; forceNewPlan=false, retain=false 2024-11-08T00:14:36,173 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bcb90ad7cb983c915be269c865806709, ASSIGN 2024-11-08T00:14:36,174 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bcb90ad7cb983c915be269c865806709, ASSIGN; state=OFFLINE, location=3302f0f507bd,38371,1731024872767; forceNewPlan=false, retain=false 2024-11-08T00:14:36,176 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=54a0ce095f1e6524de8bdf904569509f, ASSIGN 2024-11-08T00:14:36,177 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=54a0ce095f1e6524de8bdf904569509f, ASSIGN; state=OFFLINE, location=3302f0f507bd,36139,1731024872925; forceNewPlan=false, retain=false 2024-11-08T00:14:36,178 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:36,179 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN; state=OFFLINE, location=3302f0f507bd,44033,1731024872883; forceNewPlan=false, retain=false 2024-11-08T00:14:36,180 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e10c5ec7fdc210db94ac8df9187f1c4, ASSIGN 2024-11-08T00:14:36,182 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2e10c5ec7fdc210db94ac8df9187f1c4, ASSIGN; state=OFFLINE, location=3302f0f507bd,38371,1731024872767; forceNewPlan=false, retain=false 2024-11-08T00:14:36,184 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b8e8ac9592d4e973f9896a6807e10272, ASSIGN 2024-11-08T00:14:36,185 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b8e8ac9592d4e973f9896a6807e10272, ASSIGN; state=OFFLINE, location=3302f0f507bd,36139,1731024872925; forceNewPlan=false, retain=false 2024-11-08T00:14:36,186 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:36,187 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN; state=OFFLINE, location=3302f0f507bd,44033,1731024872883; forceNewPlan=false, retain=false 2024-11-08T00:14:36,189 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=850fbeeee9e2784d3676499ac7582114, ASSIGN 2024-11-08T00:14:36,190 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=850fbeeee9e2784d3676499ac7582114, ASSIGN; state=OFFLINE, location=3302f0f507bd,38371,1731024872767; forceNewPlan=false, retain=false 2024-11-08T00:14:36,191 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3bddcc13c4eccc1d32b3c09b1dcb1583, ASSIGN 2024-11-08T00:14:36,192 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3bddcc13c4eccc1d32b3c09b1dcb1583, ASSIGN; state=OFFLINE, location=3302f0f507bd,36139,1731024872925; forceNewPlan=false, retain=false 2024-11-08T00:14:36,194 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:36,195 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN; state=OFFLINE, location=3302f0f507bd,44033,1731024872883; forceNewPlan=false, retain=false 2024-11-08T00:14:36,196 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1c8336e737af786ef3c90e95f98c755f, ASSIGN 2024-11-08T00:14:36,197 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1c8336e737af786ef3c90e95f98c755f, ASSIGN; state=OFFLINE, location=3302f0f507bd,38371,1731024872767; forceNewPlan=false, retain=false 2024-11-08T00:14:36,198 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3bb82385d0b72eb69ca4bdc52663bf8a, ASSIGN 2024-11-08T00:14:36,199 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3bb82385d0b72eb69ca4bdc52663bf8a, ASSIGN; state=OFFLINE, location=3302f0f507bd,38371,1731024872767; forceNewPlan=false, retain=false 2024-11-08T00:14:36,201 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f04142b3a962ffb3cc6e9629a0aa3c6a, ASSIGN 2024-11-08T00:14:36,202 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f04142b3a962ffb3cc6e9629a0aa3c6a, ASSIGN; state=OFFLINE, location=3302f0f507bd,36139,1731024872925; forceNewPlan=false, retain=false 2024-11-08T00:14:36,204 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:36,205 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN; state=OFFLINE, location=3302f0f507bd,44033,1731024872883; forceNewPlan=false, retain=false 2024-11-08T00:14:36,206 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=13f20a01a04306fecc9b545706c9d14c, ASSIGN 2024-11-08T00:14:36,207 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=13f20a01a04306fecc9b545706c9d14c, ASSIGN; state=OFFLINE, location=3302f0f507bd,38371,1731024872767; forceNewPlan=false, retain=false 2024-11-08T00:14:36,208 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b5026dfa3bad2942533f117d619b1656, ASSIGN 2024-11-08T00:14:36,210 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b5026dfa3bad2942533f117d619b1656, ASSIGN; state=OFFLINE, location=3302f0f507bd,36139,1731024872925; forceNewPlan=false, retain=false 2024-11-08T00:14:36,211 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:36,212 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN; state=OFFLINE, location=3302f0f507bd,44033,1731024872883; forceNewPlan=false, retain=false 2024-11-08T00:14:36,213 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66bf32aac498176881371b724c9d8008, ASSIGN 2024-11-08T00:14:36,214 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=66bf32aac498176881371b724c9d8008, ASSIGN; state=OFFLINE, location=3302f0f507bd,38371,1731024872767; forceNewPlan=false, retain=false 2024-11-08T00:14:36,215 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cd2627434f69d79290b6a35838b5f326, ASSIGN 2024-11-08T00:14:36,216 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cd2627434f69d79290b6a35838b5f326, ASSIGN; state=OFFLINE, location=3302f0f507bd,36139,1731024872925; forceNewPlan=false, retain=false 2024-11-08T00:14:36,217 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ffaf29acbc927f398077e41ad232c549, ASSIGN 2024-11-08T00:14:36,218 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ffaf29acbc927f398077e41ad232c549, ASSIGN; state=OFFLINE, location=3302f0f507bd,36139,1731024872925; forceNewPlan=false, retain=false 2024-11-08T00:14:36,219 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9c5dafc4711af09b862954533a7784ca, ASSIGN 2024-11-08T00:14:36,220 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9c5dafc4711af09b862954533a7784ca, ASSIGN; state=OFFLINE, location=3302f0f507bd,38371,1731024872767; forceNewPlan=false, retain=false 2024-11-08T00:14:36,221 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:36,222 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN; state=OFFLINE, location=3302f0f507bd,44033,1731024872883; forceNewPlan=false, retain=false 2024-11-08T00:14:36,223 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8624a8443e655fa753c4de0934012d54, ASSIGN 2024-11-08T00:14:36,224 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8624a8443e655fa753c4de0934012d54, ASSIGN; state=OFFLINE, location=3302f0f507bd,36139,1731024872925; forceNewPlan=false, retain=false 2024-11-08T00:14:36,316 INFO [3302f0f507bd:36429 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-08T00:14:36,317 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c816073c18c6a09db7883f3602eda012, regionState=OPENING, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,320 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,320 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,321 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN because future has completed 2024-11-08T00:14:36,321 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=a594c54df520141421a62d3f056e15f2, regionState=OPENING, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,44033,1731024872883}] 2024-11-08T00:14:36,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN because future has completed 2024-11-08T00:14:36,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN because future has completed 2024-11-08T00:14:36,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,44033,1731024872883}] 2024-11-08T00:14:36,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN because future has completed 2024-11-08T00:14:36,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN because future has completed 2024-11-08T00:14:36,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,44033,1731024872883}] 2024-11-08T00:14:36,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN because future has completed 2024-11-08T00:14:36,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN because future has completed 2024-11-08T00:14:36,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,44033,1731024872883}] 2024-11-08T00:14:36,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN because future has completed 2024-11-08T00:14:36,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,44033,1731024872883}] 2024-11-08T00:14:36,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,44033,1731024872883}] 2024-11-08T00:14:36,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee421baf774947d334ec46882ae42d9c, server=3302f0f507bd,44033,1731024872883}] 2024-11-08T00:14:36,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,44033,1731024872883}] 2024-11-08T00:14:36,360 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f09fb3c8c48c1b2f01441065e691680c, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,361 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=bcb90ad7cb983c915be269c865806709, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,361 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,362 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=850fbeeee9e2784d3676499ac7582114, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,362 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=1c8336e737af786ef3c90e95f98c755f, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,363 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,363 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=13f20a01a04306fecc9b545706c9d14c, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f09fb3c8c48c1b2f01441065e691680c, ASSIGN because future has completed 2024-11-08T00:14:36,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f09fb3c8c48c1b2f01441065e691680c, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:36,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bcb90ad7cb983c915be269c865806709, ASSIGN because future has completed 2024-11-08T00:14:36,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2e10c5ec7fdc210db94ac8df9187f1c4, ASSIGN because future has completed 2024-11-08T00:14:36,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=850fbeeee9e2784d3676499ac7582114, ASSIGN because future has completed 2024-11-08T00:14:36,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e10c5ec7fdc210db94ac8df9187f1c4, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:36,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1c8336e737af786ef3c90e95f98c755f, ASSIGN because future has completed 2024-11-08T00:14:36,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3bb82385d0b72eb69ca4bdc52663bf8a, ASSIGN because future has completed 2024-11-08T00:14:36,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=13f20a01a04306fecc9b545706c9d14c, ASSIGN because future has completed 2024-11-08T00:14:36,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13f20a01a04306fecc9b545706c9d14c, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:36,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bb82385d0b72eb69ca4bdc52663bf8a, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:36,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c8336e737af786ef3c90e95f98c755f, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:36,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 850fbeeee9e2784d3676499ac7582114, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:36,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcb90ad7cb983c915be269c865806709, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:36,401 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=66bf32aac498176881371b724c9d8008, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,402 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9c5dafc4711af09b862954533a7784ca, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,402 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=335e583f3dd0eda08c4c42b80a743311, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,403 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=54a0ce095f1e6524de8bdf904569509f, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,404 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b8e8ac9592d4e973f9896a6807e10272, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,404 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=66bf32aac498176881371b724c9d8008, ASSIGN because future has completed 2024-11-08T00:14:36,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66bf32aac498176881371b724c9d8008, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:36,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9c5dafc4711af09b862954533a7784ca, ASSIGN because future has completed 2024-11-08T00:14:36,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=335e583f3dd0eda08c4c42b80a743311, ASSIGN because future has completed 2024-11-08T00:14:36,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c5dafc4711af09b862954533a7784ca, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:36,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=54a0ce095f1e6524de8bdf904569509f, ASSIGN because future has completed 2024-11-08T00:14:36,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b8e8ac9592d4e973f9896a6807e10272, ASSIGN because future has completed 2024-11-08T00:14:36,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3bddcc13c4eccc1d32b3c09b1dcb1583, ASSIGN because future has completed 2024-11-08T00:14:36,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bddcc13c4eccc1d32b3c09b1dcb1583, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:36,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b8e8ac9592d4e973f9896a6807e10272, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:36,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54a0ce095f1e6524de8bdf904569509f, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:36,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335e583f3dd0eda08c4c42b80a743311, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:36,438 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,438 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=b5026dfa3bad2942533f117d619b1656, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,439 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=cd2627434f69d79290b6a35838b5f326, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,439 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ffaf29acbc927f398077e41ad232c549, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,440 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8624a8443e655fa753c4de0934012d54, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f04142b3a962ffb3cc6e9629a0aa3c6a, ASSIGN because future has completed 2024-11-08T00:14:36,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f04142b3a962ffb3cc6e9629a0aa3c6a, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:36,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b5026dfa3bad2942533f117d619b1656, ASSIGN because future has completed 2024-11-08T00:14:36,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cd2627434f69d79290b6a35838b5f326, ASSIGN because future has completed 2024-11-08T00:14:36,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ffaf29acbc927f398077e41ad232c549, ASSIGN because future has completed 2024-11-08T00:14:36,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd2627434f69d79290b6a35838b5f326, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:36,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8624a8443e655fa753c4de0934012d54, ASSIGN because future has completed 2024-11-08T00:14:36,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8624a8443e655fa753c4de0934012d54, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:36,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffaf29acbc927f398077e41ad232c549, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:36,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5026dfa3bad2942533f117d619b1656, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:36,481 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:36,481 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ef74f869fdb87a20df612d1e6e2c262, NAME => 'test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-08T00:14:36,482 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,482 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,482 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,482 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,485 INFO [StoreOpener-2ef74f869fdb87a20df612d1e6e2c262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,487 INFO [StoreOpener-2ef74f869fdb87a20df612d1e6e2c262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ef74f869fdb87a20df612d1e6e2c262 columnFamilyName family 2024-11-08T00:14:36,487 DEBUG [StoreOpener-2ef74f869fdb87a20df612d1e6e2c262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,489 INFO [StoreOpener-2ef74f869fdb87a20df612d1e6e2c262-1 {}] regionserver.HStore(327): Store=2ef74f869fdb87a20df612d1e6e2c262/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,489 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,490 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,491 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,492 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,492 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,497 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,500 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2ef74f869fdb87a20df612d1e6e2c262/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,501 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 2ef74f869fdb87a20df612d1e6e2c262; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68591416, jitterRate=0.02209174633026123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,502 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:36,502 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 2ef74f869fdb87a20df612d1e6e2c262: Running coprocessor pre-open hook at 1731024876483Writing region info on filesystem at 1731024876483Initializing all the Stores at 1731024876484 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876484Cleaning up temporary data from old regions at 1731024876492 (+8 ms)Running coprocessor post-open hooks at 1731024876502 (+10 ms)Region opened successfully at 1731024876502 2024-11-08T00:14:36,504 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262., pid=36, masterSystemTime=1731024876475 2024-11-08T00:14:36,507 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:36,508 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:36,508 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. 2024-11-08T00:14:36,508 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 8280a310195422b4e0184bc7c709a2e9, NAME => 'test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-08T00:14:36,509 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,509 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,509 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,509 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,509 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,44033,1731024872883 because future has completed 2024-11-08T00:14:36,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=14 2024-11-08T00:14:36,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,44033,1731024872883 in 168 msec 2024-11-08T00:14:36,521 INFO [StoreOpener-8280a310195422b4e0184bc7c709a2e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN in 368 msec 2024-11-08T00:14:36,523 INFO [StoreOpener-8280a310195422b4e0184bc7c709a2e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8280a310195422b4e0184bc7c709a2e9 columnFamilyName family 2024-11-08T00:14:36,523 DEBUG [StoreOpener-8280a310195422b4e0184bc7c709a2e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,525 INFO [StoreOpener-8280a310195422b4e0184bc7c709a2e9-1 {}] regionserver.HStore(327): Store=8280a310195422b4e0184bc7c709a2e9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,525 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,526 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,527 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,529 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T00:14:36,529 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,529 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,533 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50081, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T00:14:36,537 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/8280a310195422b4e0184bc7c709a2e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,538 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 8280a310195422b4e0184bc7c709a2e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70524143, jitterRate=0.05089162290096283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,538 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:36,538 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 8280a310195422b4e0184bc7c709a2e9: Running coprocessor pre-open hook at 1731024876509Writing region info on filesystem at 1731024876509Initializing all the Stores at 1731024876511 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876511Cleaning up temporary data from old regions at 1731024876529 (+18 ms)Running coprocessor post-open hooks at 1731024876538 (+9 ms)Region opened successfully at 1731024876538 2024-11-08T00:14:36,540 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9., pid=38, masterSystemTime=1731024876475 2024-11-08T00:14:36,540 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:36,540 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e10c5ec7fdc210db94ac8df9187f1c4, NAME => 'test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-08T00:14:36,541 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,541 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,541 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,541 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,543 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. 2024-11-08T00:14:36,544 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. 2024-11-08T00:14:36,544 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:36,544 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 7acb38c712f4cad61cf5f78f7ffa9a18, NAME => 'test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-08T00:14:36,545 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8280a310195422b4e0184bc7c709a2e9, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,545 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,545 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,545 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,545 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,44033,1731024872883 because future has completed 2024-11-08T00:14:36,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=24 2024-11-08T00:14:36,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,44033,1731024872883 in 195 msec 2024-11-08T00:14:36,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN in 405 msec 2024-11-08T00:14:36,564 INFO [StoreOpener-2e10c5ec7fdc210db94ac8df9187f1c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,565 INFO [StoreOpener-7acb38c712f4cad61cf5f78f7ffa9a18-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,568 INFO [StoreOpener-2e10c5ec7fdc210db94ac8df9187f1c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e10c5ec7fdc210db94ac8df9187f1c4 columnFamilyName family 2024-11-08T00:14:36,568 DEBUG [StoreOpener-2e10c5ec7fdc210db94ac8df9187f1c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,569 INFO [StoreOpener-2e10c5ec7fdc210db94ac8df9187f1c4-1 {}] regionserver.HStore(327): Store=2e10c5ec7fdc210db94ac8df9187f1c4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,570 INFO [StoreOpener-7acb38c712f4cad61cf5f78f7ffa9a18-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7acb38c712f4cad61cf5f78f7ffa9a18 columnFamilyName family 2024-11-08T00:14:36,570 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,570 DEBUG [StoreOpener-7acb38c712f4cad61cf5f78f7ffa9a18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,571 INFO [StoreOpener-7acb38c712f4cad61cf5f78f7ffa9a18-1 {}] regionserver.HStore(327): Store=7acb38c712f4cad61cf5f78f7ffa9a18/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,572 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,572 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T00:14:36,573 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56207, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T00:14:36,574 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,575 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,575 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,575 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,576 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,579 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,579 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,581 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,587 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,602 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/7acb38c712f4cad61cf5f78f7ffa9a18/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,603 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 7acb38c712f4cad61cf5f78f7ffa9a18; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59572183, jitterRate=-0.11230529844760895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,604 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:36,604 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 7acb38c712f4cad61cf5f78f7ffa9a18: Running coprocessor pre-open hook at 1731024876545Writing region info on filesystem at 1731024876545Initializing all the Stores at 1731024876548 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876548Cleaning up temporary data from old regions at 1731024876575 (+27 ms)Running coprocessor post-open hooks at 1731024876604 (+29 ms)Region opened successfully at 1731024876604 2024-11-08T00:14:36,605 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:36,605 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => b8e8ac9592d4e973f9896a6807e10272, NAME => 'test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-08T00:14:36,605 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,606 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,606 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,606 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,606 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18., pid=33, masterSystemTime=1731024876475 2024-11-08T00:14:36,610 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:36,610 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:36,610 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c. 2024-11-08T00:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => ee421baf774947d334ec46882ae42d9c, NAME => 'test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-08T00:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,611 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,611 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,44033,1731024872883 because future has completed 2024-11-08T00:14:36,617 INFO [StoreOpener-ee421baf774947d334ec46882ae42d9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,619 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2e10c5ec7fdc210db94ac8df9187f1c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,619 INFO [StoreOpener-b8e8ac9592d4e973f9896a6807e10272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,620 INFO [StoreOpener-ee421baf774947d334ec46882ae42d9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee421baf774947d334ec46882ae42d9c columnFamilyName family 2024-11-08T00:14:36,620 DEBUG [StoreOpener-ee421baf774947d334ec46882ae42d9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,620 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 2e10c5ec7fdc210db94ac8df9187f1c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69131100, jitterRate=0.03013366460800171}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,620 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:36,621 INFO [StoreOpener-ee421baf774947d334ec46882ae42d9c-1 {}] regionserver.HStore(327): Store=ee421baf774947d334ec46882ae42d9c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,621 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 2e10c5ec7fdc210db94ac8df9187f1c4: Running coprocessor pre-open hook at 1731024876542Writing region info on filesystem at 1731024876542Initializing all the Stores at 1731024876544 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876544Cleaning up temporary data from old regions at 1731024876579 (+35 ms)Running coprocessor post-open hooks at 1731024876620 (+41 ms)Region opened successfully at 1731024876621 (+1 ms) 2024-11-08T00:14:36,621 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,622 INFO [StoreOpener-b8e8ac9592d4e973f9896a6807e10272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8e8ac9592d4e973f9896a6807e10272 columnFamilyName family 2024-11-08T00:14:36,622 DEBUG [StoreOpener-b8e8ac9592d4e973f9896a6807e10272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,622 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4., pid=40, masterSystemTime=1731024876528 2024-11-08T00:14:36,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=21 2024-11-08T00:14:36,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,44033,1731024872883 in 285 msec 2024-11-08T00:14:36,623 INFO [StoreOpener-b8e8ac9592d4e973f9896a6807e10272-1 {}] regionserver.HStore(327): Store=b8e8ac9592d4e973f9896a6807e10272/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,623 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN in 472 msec 2024-11-08T00:14:36,626 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,627 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,627 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:36,628 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:36,628 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,628 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,628 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:36,628 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 850fbeeee9e2784d3676499ac7582114, NAME => 'test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-08T00:14:36,629 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,629 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,629 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,629 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,629 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,630 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,631 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,632 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,632 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e10c5ec7fdc210db94ac8df9187f1c4, server=3302f0f507bd,38371,1731024872767 because future has completed 2024-11-08T00:14:36,632 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,637 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=23 2024-11-08T00:14:36,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e10c5ec7fdc210db94ac8df9187f1c4, server=3302f0f507bd,38371,1731024872767 in 265 msec 2024-11-08T00:14:36,640 INFO [StoreOpener-850fbeeee9e2784d3676499ac7582114-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,641 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/b8e8ac9592d4e973f9896a6807e10272/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,642 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/ee421baf774947d334ec46882ae42d9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2e10c5ec7fdc210db94ac8df9187f1c4, ASSIGN in 489 msec 2024-11-08T00:14:36,643 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened b8e8ac9592d4e973f9896a6807e10272; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68855151, jitterRate=0.026021704077720642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,643 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:36,643 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for b8e8ac9592d4e973f9896a6807e10272: Running coprocessor pre-open hook at 1731024876608Writing region info on filesystem at 1731024876608Initializing all the Stores at 1731024876610 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876610Cleaning up temporary data from old regions at 1731024876628 (+18 ms)Running coprocessor post-open hooks at 1731024876643 (+15 ms)Region opened successfully at 1731024876643 2024-11-08T00:14:36,643 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened ee421baf774947d334ec46882ae42d9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73113415, jitterRate=0.08947478234767914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,643 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee421baf774947d334ec46882ae42d9c 2024-11-08T00:14:36,644 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for ee421baf774947d334ec46882ae42d9c: Running coprocessor pre-open hook at 1731024876611Writing region info on filesystem at 1731024876611Initializing all the Stores at 1731024876614 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876614Cleaning up temporary data from old regions at 1731024876632 (+18 ms)Running coprocessor post-open hooks at 1731024876643 (+11 ms)Region opened successfully at 1731024876643 2024-11-08T00:14:36,644 INFO [StoreOpener-850fbeeee9e2784d3676499ac7582114-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850fbeeee9e2784d3676499ac7582114 columnFamilyName family 2024-11-08T00:14:36,644 DEBUG [StoreOpener-850fbeeee9e2784d3676499ac7582114-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,645 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c., pid=37, masterSystemTime=1731024876475 2024-11-08T00:14:36,645 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272., pid=49, masterSystemTime=1731024876571 2024-11-08T00:14:36,645 INFO [StoreOpener-850fbeeee9e2784d3676499ac7582114-1 {}] regionserver.HStore(327): Store=850fbeeee9e2784d3676499ac7582114/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,645 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,647 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,648 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,648 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c. 2024-11-08T00:14:36,648 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,nnn,1731024875468.ee421baf774947d334ec46882ae42d9c. 2024-11-08T00:14:36,648 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8. 2024-11-08T00:14:36,649 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => faad0b2f1a2327cef2cd0aefa9a4d1c8, NAME => 'test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-08T00:14:36,649 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,649 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,649 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,649 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,650 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=ee421baf774947d334ec46882ae42d9c, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,650 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,650 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,652 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:36,653 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:36,653 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:36,654 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => cd2627434f69d79290b6a35838b5f326, NAME => 'test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-08T00:14:36,654 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,654 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,654 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,654 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,655 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee421baf774947d334ec46882ae42d9c, server=3302f0f507bd,44033,1731024872883 because future has completed 2024-11-08T00:14:36,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:14:36,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b8e8ac9592d4e973f9896a6807e10272, server=3302f0f507bd,36139,1731024872925 because future has completed 2024-11-08T00:14:36,660 INFO [StoreOpener-faad0b2f1a2327cef2cd0aefa9a4d1c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,662 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/850fbeeee9e2784d3676499ac7582114/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,663 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 850fbeeee9e2784d3676499ac7582114; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73265080, jitterRate=0.09173476696014404}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,663 INFO [StoreOpener-cd2627434f69d79290b6a35838b5f326-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,663 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:36,665 INFO [StoreOpener-faad0b2f1a2327cef2cd0aefa9a4d1c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region faad0b2f1a2327cef2cd0aefa9a4d1c8 columnFamilyName family 2024-11-08T00:14:36,665 DEBUG [StoreOpener-faad0b2f1a2327cef2cd0aefa9a4d1c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,666 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 850fbeeee9e2784d3676499ac7582114: Running coprocessor pre-open hook at 1731024876629Writing region info on filesystem at 1731024876629Initializing all the Stores at 1731024876632 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876632Cleaning up temporary data from old regions at 1731024876649 (+17 ms)Running coprocessor post-open hooks at 1731024876663 (+14 ms)Region opened successfully at 1731024876663 2024-11-08T00:14:36,667 INFO [StoreOpener-faad0b2f1a2327cef2cd0aefa9a4d1c8-1 {}] regionserver.HStore(327): Store=faad0b2f1a2327cef2cd0aefa9a4d1c8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,668 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,670 INFO [StoreOpener-cd2627434f69d79290b6a35838b5f326-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd2627434f69d79290b6a35838b5f326 columnFamilyName family 2024-11-08T00:14:36,670 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,670 DEBUG [StoreOpener-cd2627434f69d79290b6a35838b5f326-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,670 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114., pid=44, masterSystemTime=1731024876528 2024-11-08T00:14:36,671 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,671 INFO [StoreOpener-cd2627434f69d79290b6a35838b5f326-1 {}] regionserver.HStore(327): Store=cd2627434f69d79290b6a35838b5f326/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,672 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,672 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,674 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:36,674 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:36,674 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:36,675 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c5dafc4711af09b862954533a7784ca, NAME => 'test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-08T00:14:36,675 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,675 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,675 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,675 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,675 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=18 2024-11-08T00:14:36,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure ee421baf774947d334ec46882ae42d9c, server=3302f0f507bd,44033,1731024872883 in 308 msec 2024-11-08T00:14:36,676 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,677 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,678 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,680 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,680 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,681 INFO [StoreOpener-9c5dafc4711af09b862954533a7784ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN in 525 msec 2024-11-08T00:14:36,684 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/faad0b2f1a2327cef2cd0aefa9a4d1c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,685 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,685 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened faad0b2f1a2327cef2cd0aefa9a4d1c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69623713, jitterRate=0.037474170327186584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,685 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for faad0b2f1a2327cef2cd0aefa9a4d1c8 2024-11-08T00:14:36,685 INFO [StoreOpener-9c5dafc4711af09b862954533a7784ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c5dafc4711af09b862954533a7784ca columnFamilyName family 2024-11-08T00:14:36,685 DEBUG [StoreOpener-9c5dafc4711af09b862954533a7784ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,685 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for faad0b2f1a2327cef2cd0aefa9a4d1c8: Running coprocessor pre-open hook at 1731024876650Writing region info on filesystem at 1731024876650Initializing all the Stores at 1731024876653 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876653Cleaning up temporary data from old regions at 1731024876672 (+19 ms)Running coprocessor post-open hooks at 1731024876685 (+13 ms)Region opened successfully at 1731024876685 2024-11-08T00:14:36,687 INFO [StoreOpener-9c5dafc4711af09b862954533a7784ca-1 {}] regionserver.HStore(327): Store=9c5dafc4711af09b862954533a7784ca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,687 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8., pid=34, masterSystemTime=1731024876475 2024-11-08T00:14:36,688 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 850fbeeee9e2784d3676499ac7582114, server=3302f0f507bd,38371,1731024872767 because future has completed 2024-11-08T00:14:36,690 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,691 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8. 2024-11-08T00:14:36,691 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,691 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ggg,1731024875468.faad0b2f1a2327cef2cd0aefa9a4d1c8. 2024-11-08T00:14:36,692 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012. 2024-11-08T00:14:36,692 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => c816073c18c6a09db7883f3602eda012, NAME => 'test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-08T00:14:36,692 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,693 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,693 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,693 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,693 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,695 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,695 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=22 2024-11-08T00:14:36,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure b8e8ac9592d4e973f9896a6807e10272, server=3302f0f507bd,36139,1731024872925 in 268 msec 2024-11-08T00:14:36,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b8e8ac9592d4e973f9896a6807e10272, ASSIGN in 549 msec 2024-11-08T00:14:36,705 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/cd2627434f69d79290b6a35838b5f326/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,706 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened cd2627434f69d79290b6a35838b5f326; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60293656, jitterRate=-0.1015545129776001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,707 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:36,707 INFO [StoreOpener-c816073c18c6a09db7883f3602eda012-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,707 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for cd2627434f69d79290b6a35838b5f326: Running coprocessor pre-open hook at 1731024876655Writing region info on filesystem at 1731024876655Initializing all the Stores at 1731024876658 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876658Cleaning up temporary data from old regions at 1731024876680 (+22 ms)Running coprocessor post-open hooks at 1731024876707 (+27 ms)Region opened successfully at 1731024876707 2024-11-08T00:14:36,707 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,708 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,708 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326., pid=53, masterSystemTime=1731024876571 2024-11-08T00:14:36,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,44033,1731024872883 because future has completed 2024-11-08T00:14:36,713 INFO [StoreOpener-c816073c18c6a09db7883f3602eda012-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c816073c18c6a09db7883f3602eda012 columnFamilyName family 2024-11-08T00:14:36,714 DEBUG [StoreOpener-c816073c18c6a09db7883f3602eda012-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,714 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:36,714 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:36,714 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:36,715 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => ffaf29acbc927f398077e41ad232c549, NAME => 'test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-08T00:14:36,715 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,715 INFO [StoreOpener-c816073c18c6a09db7883f3602eda012-1 {}] regionserver.HStore(327): Store=c816073c18c6a09db7883f3602eda012/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,715 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,715 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,715 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,715 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,717 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/9c5dafc4711af09b862954533a7784ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,717 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,717 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 9c5dafc4711af09b862954533a7784ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69522947, jitterRate=0.03597263991832733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,717 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:36,718 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 9c5dafc4711af09b862954533a7784ca: Running coprocessor pre-open hook at 1731024876675Writing region info on filesystem at 1731024876675Initializing all the Stores at 1731024876678 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876678Cleaning up temporary data from old regions at 1731024876695 (+17 ms)Running coprocessor post-open hooks at 1731024876717 (+22 ms)Region opened successfully at 1731024876718 (+1 ms) 2024-11-08T00:14:36,718 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,719 INFO [StoreOpener-ffaf29acbc927f398077e41ad232c549-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,719 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,719 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,721 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca., pid=47, masterSystemTime=1731024876528 2024-11-08T00:14:36,721 INFO [StoreOpener-ffaf29acbc927f398077e41ad232c549-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffaf29acbc927f398077e41ad232c549 columnFamilyName family 2024-11-08T00:14:36,722 DEBUG [StoreOpener-ffaf29acbc927f398077e41ad232c549-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=20 2024-11-08T00:14:36,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 850fbeeee9e2784d3676499ac7582114, server=3302f0f507bd,38371,1731024872767 in 325 msec 2024-11-08T00:14:36,724 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:36,725 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:36,725 INFO [StoreOpener-ffaf29acbc927f398077e41ad232c549-1 {}] regionserver.HStore(327): Store=ffaf29acbc927f398077e41ad232c549/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,725 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:36,725 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=3302f0f507bd,44033,1731024872883, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T00:14:36,725 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,725 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 13f20a01a04306fecc9b545706c9d14c, NAME => 'test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-08T00:14:36,725 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,725 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,726 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,726 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,727 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,728 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=850fbeeee9e2784d3676499ac7582114, ASSIGN in 572 msec 2024-11-08T00:14:36,729 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,730 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,730 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,732 INFO [StoreOpener-13f20a01a04306fecc9b545706c9d14c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,733 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,734 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,738 INFO [StoreOpener-13f20a01a04306fecc9b545706c9d14c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13f20a01a04306fecc9b545706c9d14c columnFamilyName family 2024-11-08T00:14:36,739 DEBUG [StoreOpener-13f20a01a04306fecc9b545706c9d14c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c5dafc4711af09b862954533a7784ca, server=3302f0f507bd,38371,1731024872767 because future has completed 2024-11-08T00:14:36,740 INFO [StoreOpener-13f20a01a04306fecc9b545706c9d14c-1 {}] regionserver.HStore(327): Store=13f20a01a04306fecc9b545706c9d14c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,740 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,742 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,742 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,743 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,743 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd2627434f69d79290b6a35838b5f326, server=3302f0f507bd,36139,1731024872925 because future has completed 2024-11-08T00:14:36,748 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/c816073c18c6a09db7883f3602eda012/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,749 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=7 2024-11-08T00:14:36,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c5dafc4711af09b862954533a7784ca, server=3302f0f507bd,38371,1731024872767 in 332 msec 2024-11-08T00:14:36,753 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened c816073c18c6a09db7883f3602eda012; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61919902, jitterRate=-0.07732155919075012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,753 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c816073c18c6a09db7883f3602eda012 2024-11-08T00:14:36,753 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for c816073c18c6a09db7883f3602eda012: Running coprocessor pre-open hook at 1731024876693Writing region info on filesystem at 1731024876693Initializing all the Stores at 1731024876705 (+12 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876706 (+1 ms)Cleaning up temporary data from old regions at 1731024876719 (+13 ms)Running coprocessor post-open hooks at 1731024876753 (+34 ms)Region opened successfully at 1731024876753 2024-11-08T00:14:36,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9c5dafc4711af09b862954533a7784ca, ASSIGN in 601 msec 2024-11-08T00:14:36,760 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012., pid=31, masterSystemTime=1731024876475 2024-11-08T00:14:36,761 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/ffaf29acbc927f398077e41ad232c549/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,762 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/13f20a01a04306fecc9b545706c9d14c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,762 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened ffaf29acbc927f398077e41ad232c549; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70827265, jitterRate=0.05540849268436432}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,763 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:36,763 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for ffaf29acbc927f398077e41ad232c549: Running coprocessor pre-open hook at 1731024876716Writing region info on filesystem at 1731024876716Initializing all the Stores at 1731024876717 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876717Cleaning up temporary data from old regions at 1731024876730 (+13 ms)Running coprocessor post-open hooks at 1731024876763 (+33 ms)Region opened successfully at 1731024876763 2024-11-08T00:14:36,764 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 13f20a01a04306fecc9b545706c9d14c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71442206, jitterRate=0.06457182765007019}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,764 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:36,764 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 13f20a01a04306fecc9b545706c9d14c: Running coprocessor pre-open hook at 1731024876726Writing region info on filesystem at 1731024876726Initializing all the Stores at 1731024876727 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876727Cleaning up temporary data from old regions at 1731024876744 (+17 ms)Running coprocessor post-open hooks at 1731024876764 (+20 ms)Region opened successfully at 1731024876764 2024-11-08T00:14:36,765 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549., pid=55, masterSystemTime=1731024876571 2024-11-08T00:14:36,765 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c., pid=41, masterSystemTime=1731024876528 2024-11-08T00:14:36,769 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012. 2024-11-08T00:14:36,769 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1731024875468.c816073c18c6a09db7883f3602eda012. 2024-11-08T00:14:36,769 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:36,769 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 40e3fed5cf6cc7497a1c2220e1de9b9b, NAME => 'test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-08T00:14:36,770 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,770 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,770 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,770 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,772 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:36,772 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:36,772 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:36,772 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 3bddcc13c4eccc1d32b3c09b1dcb1583, NAME => 'test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-08T00:14:36,772 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,773 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,773 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,773 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=9 2024-11-08T00:14:36,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure cd2627434f69d79290b6a35838b5f326, server=3302f0f507bd,36139,1731024872925 in 316 msec 2024-11-08T00:14:36,774 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:36,774 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:36,775 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:36,775 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 3bb82385d0b72eb69ca4bdc52663bf8a, NAME => 'test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-08T00:14:36,775 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,776 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,776 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,776 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cd2627434f69d79290b6a35838b5f326, ASSIGN in 622 msec 2024-11-08T00:14:36,780 INFO [StoreOpener-40e3fed5cf6cc7497a1c2220e1de9b9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,782 INFO [StoreOpener-3bddcc13c4eccc1d32b3c09b1dcb1583-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,782 INFO [StoreOpener-3bb82385d0b72eb69ca4bdc52663bf8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,786 INFO [StoreOpener-3bddcc13c4eccc1d32b3c09b1dcb1583-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bddcc13c4eccc1d32b3c09b1dcb1583 columnFamilyName family 2024-11-08T00:14:36,786 DEBUG [StoreOpener-3bddcc13c4eccc1d32b3c09b1dcb1583-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13f20a01a04306fecc9b545706c9d14c, server=3302f0f507bd,38371,1731024872767 because future has completed 2024-11-08T00:14:36,787 INFO [StoreOpener-40e3fed5cf6cc7497a1c2220e1de9b9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40e3fed5cf6cc7497a1c2220e1de9b9b columnFamilyName family 2024-11-08T00:14:36,788 DEBUG [StoreOpener-40e3fed5cf6cc7497a1c2220e1de9b9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,788 INFO [StoreOpener-3bb82385d0b72eb69ca4bdc52663bf8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bb82385d0b72eb69ca4bdc52663bf8a columnFamilyName family 2024-11-08T00:14:36,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,788 DEBUG [StoreOpener-3bb82385d0b72eb69ca4bdc52663bf8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,789 INFO [StoreOpener-3bddcc13c4eccc1d32b3c09b1dcb1583-1 {}] regionserver.HStore(327): Store=3bddcc13c4eccc1d32b3c09b1dcb1583/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,789 INFO [StoreOpener-40e3fed5cf6cc7497a1c2220e1de9b9b-1 {}] regionserver.HStore(327): Store=40e3fed5cf6cc7497a1c2220e1de9b9b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,790 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,790 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,791 INFO [StoreOpener-3bb82385d0b72eb69ca4bdc52663bf8a-1 {}] regionserver.HStore(327): Store=3bb82385d0b72eb69ca4bdc52663bf8a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,791 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,791 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,791 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffaf29acbc927f398077e41ad232c549, server=3302f0f507bd,36139,1731024872925 because future has completed 2024-11-08T00:14:36,793 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,793 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,793 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,793 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,793 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,793 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,793 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,796 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,797 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,797 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,798 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,798 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=13 2024-11-08T00:14:36,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 13f20a01a04306fecc9b545706c9d14c, server=3302f0f507bd,38371,1731024872767 in 418 msec 2024-11-08T00:14:36,804 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/3bddcc13c4eccc1d32b3c09b1dcb1583/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,804 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/40e3fed5cf6cc7497a1c2220e1de9b9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,805 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 40e3fed5cf6cc7497a1c2220e1de9b9b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60422376, jitterRate=-0.09963643550872803}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,805 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:36,805 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 3bddcc13c4eccc1d32b3c09b1dcb1583; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70389724, jitterRate=0.04888862371444702}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,805 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 40e3fed5cf6cc7497a1c2220e1de9b9b: Running coprocessor pre-open hook at 1731024876770Writing region info on filesystem at 1731024876770Initializing all the Stores at 1731024876778 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876778Cleaning up temporary data from old regions at 1731024876794 (+16 ms)Running coprocessor post-open hooks at 1731024876805 (+11 ms)Region opened successfully at 1731024876805 2024-11-08T00:14:36,805 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:36,806 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 3bddcc13c4eccc1d32b3c09b1dcb1583: Running coprocessor pre-open hook at 1731024876773Writing region info on filesystem at 1731024876773Initializing all the Stores at 1731024876778 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876778Cleaning up temporary data from old regions at 1731024876794 (+16 ms)Running coprocessor post-open hooks at 1731024876805 (+11 ms)Region opened successfully at 1731024876805 2024-11-08T00:14:36,807 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b., pid=32, masterSystemTime=1731024876475 2024-11-08T00:14:36,807 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583., pid=48, masterSystemTime=1731024876571 2024-11-08T00:14:36,808 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=3302f0f507bd,36139,1731024872925, table=test, region=ffaf29acbc927f398077e41ad232c549. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T00:14:36,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=13f20a01a04306fecc9b545706c9d14c, ASSIGN in 648 msec 2024-11-08T00:14:36,809 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,810 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:36,811 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:36,811 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:36,811 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => a594c54df520141421a62d3f056e15f2, NAME => 'test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-08T00:14:36,812 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,812 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,812 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,812 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,814 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:36,814 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:36,814 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:36,814 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => b5026dfa3bad2942533f117d619b1656, NAME => 'test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-08T00:14:36,815 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,815 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,815 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,815 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=8 2024-11-08T00:14:36,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ffaf29acbc927f398077e41ad232c549, server=3302f0f507bd,36139,1731024872925 in 354 msec 2024-11-08T00:14:36,822 INFO [StoreOpener-a594c54df520141421a62d3f056e15f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,824 INFO [StoreOpener-a594c54df520141421a62d3f056e15f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a594c54df520141421a62d3f056e15f2 columnFamilyName family 2024-11-08T00:14:36,824 DEBUG [StoreOpener-a594c54df520141421a62d3f056e15f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,825 INFO [StoreOpener-b5026dfa3bad2942533f117d619b1656-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,829 INFO [StoreOpener-a594c54df520141421a62d3f056e15f2-1 {}] regionserver.HStore(327): Store=a594c54df520141421a62d3f056e15f2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,829 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,830 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=3302f0f507bd,44033,1731024872883, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T00:14:36,831 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,832 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,833 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/3bb82385d0b72eb69ca4bdc52663bf8a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,833 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,833 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ffaf29acbc927f398077e41ad232c549, ASSIGN in 669 msec 2024-11-08T00:14:36,836 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 3bb82385d0b72eb69ca4bdc52663bf8a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73034737, jitterRate=0.08830238878726959}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,836 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:36,836 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 3bb82385d0b72eb69ca4bdc52663bf8a: Running coprocessor pre-open hook at 1731024876776Writing region info on filesystem at 1731024876776Initializing all the Stores at 1731024876781 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876781Cleaning up temporary data from old regions at 1731024876798 (+17 ms)Running coprocessor post-open hooks at 1731024876836 (+38 ms)Region opened successfully at 1731024876836 2024-11-08T00:14:36,837 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a., pid=42, masterSystemTime=1731024876528 2024-11-08T00:14:36,838 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,839 INFO [StoreOpener-b5026dfa3bad2942533f117d619b1656-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5026dfa3bad2942533f117d619b1656 columnFamilyName family 2024-11-08T00:14:36,839 DEBUG [StoreOpener-b5026dfa3bad2942533f117d619b1656-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,842 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:36,842 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=48, ppid=19, state=RUNNABLE, hasLock=true; OpenRegionProcedure 3bddcc13c4eccc1d32b3c09b1dcb1583, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,842 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:36,842 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:36,843 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c8336e737af786ef3c90e95f98c755f, NAME => 'test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-08T00:14:36,843 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,843 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,843 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,844 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,844 INFO [StoreOpener-b5026dfa3bad2942533f117d619b1656-1 {}] regionserver.HStore(327): Store=b5026dfa3bad2942533f117d619b1656/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,844 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=19 2024-11-08T00:14:36,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 3bddcc13c4eccc1d32b3c09b1dcb1583, server=3302f0f507bd,36139,1731024872925 in 424 msec 2024-11-08T00:14:36,847 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,850 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3bddcc13c4eccc1d32b3c09b1dcb1583, ASSIGN in 695 msec 2024-11-08T00:14:36,850 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,851 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,851 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,853 INFO [StoreOpener-1c8336e737af786ef3c90e95f98c755f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,854 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bb82385d0b72eb69ca4bdc52663bf8a, server=3302f0f507bd,38371,1731024872767 because future has completed 2024-11-08T00:14:36,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,859 INFO [StoreOpener-1c8336e737af786ef3c90e95f98c755f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c8336e737af786ef3c90e95f98c755f columnFamilyName family 2024-11-08T00:14:36,859 DEBUG [StoreOpener-1c8336e737af786ef3c90e95f98c755f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,44033,1731024872883 because future has completed 2024-11-08T00:14:36,866 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/b5026dfa3bad2942533f117d619b1656/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,867 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened b5026dfa3bad2942533f117d619b1656; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60550419, jitterRate=-0.09772844612598419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,867 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:36,867 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for b5026dfa3bad2942533f117d619b1656: Running coprocessor pre-open hook at 1731024876815Writing region info on filesystem at 1731024876815Initializing all the Stores at 1731024876817 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876817Cleaning up temporary data from old regions at 1731024876851 (+34 ms)Running coprocessor post-open hooks at 1731024876867 (+16 ms)Region opened successfully at 1731024876867 2024-11-08T00:14:36,869 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656., pid=56, masterSystemTime=1731024876571 2024-11-08T00:14:36,870 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/a594c54df520141421a62d3f056e15f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,870 INFO [StoreOpener-1c8336e737af786ef3c90e95f98c755f-1 {}] regionserver.HStore(327): Store=1c8336e737af786ef3c90e95f98c755f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,871 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,872 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=16 2024-11-08T00:14:36,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 3bb82385d0b72eb69ca4bdc52663bf8a, server=3302f0f507bd,38371,1731024872767 in 483 msec 2024-11-08T00:14:36,874 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:36,874 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:36,873 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened a594c54df520141421a62d3f056e15f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60608498, jitterRate=-0.09686300158500671}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,874 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:36,874 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:36,874 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for a594c54df520141421a62d3f056e15f2: Running coprocessor pre-open hook at 1731024876812Writing region info on filesystem at 1731024876812Initializing all the Stores at 1731024876815 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876815Cleaning up temporary data from old regions at 1731024876834 (+19 ms)Running coprocessor post-open hooks at 1731024876874 (+40 ms)Region opened successfully at 1731024876874 2024-11-08T00:14:36,874 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 8624a8443e655fa753c4de0934012d54, NAME => 'test,,1731024875468.8624a8443e655fa753c4de0934012d54.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-08T00:14:36,875 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,875 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,,1731024875468.8624a8443e655fa753c4de0934012d54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,875 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,875 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,875 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2., pid=35, masterSystemTime=1731024876475 2024-11-08T00:14:36,876 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,877 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,877 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,880 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,882 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:36,882 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:36,882 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5026dfa3bad2942533f117d619b1656, server=3302f0f507bd,36139,1731024872925 because future has completed 2024-11-08T00:14:36,892 INFO [StoreOpener-8624a8443e655fa753c4de0934012d54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,895 INFO [StoreOpener-8624a8443e655fa753c4de0934012d54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8624a8443e655fa753c4de0934012d54 columnFamilyName family 2024-11-08T00:14:36,895 DEBUG [StoreOpener-8624a8443e655fa753c4de0934012d54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=12 2024-11-08T00:14:36,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure b5026dfa3bad2942533f117d619b1656, server=3302f0f507bd,36139,1731024872925 in 431 msec 2024-11-08T00:14:36,897 INFO [StoreOpener-8624a8443e655fa753c4de0934012d54-1 {}] regionserver.HStore(327): Store=8624a8443e655fa753c4de0934012d54/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,898 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,899 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,900 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,901 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,901 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b5026dfa3bad2942533f117d619b1656, ASSIGN in 747 msec 2024-11-08T00:14:36,903 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=a594c54df520141421a62d3f056e15f2, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:36,905 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/1c8336e737af786ef3c90e95f98c755f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,907 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 1c8336e737af786ef3c90e95f98c755f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69591553, jitterRate=0.036994948983192444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,907 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:36,907 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 1c8336e737af786ef3c90e95f98c755f: Running coprocessor pre-open hook at 1731024876844Writing region info on filesystem at 1731024876844Initializing all the Stores at 1731024876846 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876846Cleaning up temporary data from old regions at 1731024876877 (+31 ms)Running coprocessor post-open hooks at 1731024876907 (+30 ms)Region opened successfully at 1731024876907 2024-11-08T00:14:36,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,44033,1731024872883 because future has completed 2024-11-08T00:14:36,909 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f., pid=43, masterSystemTime=1731024876528 2024-11-08T00:14:36,912 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/8624a8443e655fa753c4de0934012d54/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,913 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:36,913 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:36,913 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:36,913 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 8624a8443e655fa753c4de0934012d54; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73263809, jitterRate=0.09171582758426666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,914 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:36,914 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 8624a8443e655fa753c4de0934012d54: Running coprocessor pre-open hook at 1731024876876Writing region info on filesystem at 1731024876876Initializing all the Stores at 1731024876879 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876879Cleaning up temporary data from old regions at 1731024876901 (+22 ms)Running coprocessor post-open hooks at 1731024876914 (+13 ms)Region opened successfully at 1731024876914 2024-11-08T00:14:36,914 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => bcb90ad7cb983c915be269c865806709, NAME => 'test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-08T00:14:36,914 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,914 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,914 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,914 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,915 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731024875468.8624a8443e655fa753c4de0934012d54., pid=54, masterSystemTime=1731024876571 2024-11-08T00:14:36,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=6 2024-11-08T00:14:36,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,44033,1731024872883 in 569 msec 2024-11-08T00:14:36,919 INFO [StoreOpener-bcb90ad7cb983c915be269c865806709-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,919 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:36,922 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:36,922 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:36,922 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => f04142b3a962ffb3cc6e9629a0aa3c6a, NAME => 'test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-08T00:14:36,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8624a8443e655fa753c4de0934012d54, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,922 INFO [StoreOpener-bcb90ad7cb983c915be269c865806709-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcb90ad7cb983c915be269c865806709 columnFamilyName family 2024-11-08T00:14:36,922 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,922 DEBUG [StoreOpener-bcb90ad7cb983c915be269c865806709-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,922 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,923 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,923 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,923 INFO [StoreOpener-bcb90ad7cb983c915be269c865806709-1 {}] regionserver.HStore(327): Store=bcb90ad7cb983c915be269c865806709/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,924 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,925 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,926 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,926 INFO [StoreOpener-f04142b3a962ffb3cc6e9629a0aa3c6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,927 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,927 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8624a8443e655fa753c4de0934012d54, server=3302f0f507bd,36139,1731024872925 because future has completed 2024-11-08T00:14:36,930 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,931 INFO [StoreOpener-f04142b3a962ffb3cc6e9629a0aa3c6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f04142b3a962ffb3cc6e9629a0aa3c6a columnFamilyName family 2024-11-08T00:14:36,931 DEBUG [StoreOpener-f04142b3a962ffb3cc6e9629a0aa3c6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,931 INFO [StoreOpener-f04142b3a962ffb3cc6e9629a0aa3c6a-1 {}] regionserver.HStore(327): Store=f04142b3a962ffb3cc6e9629a0aa3c6a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,931 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN in 775 msec 2024-11-08T00:14:36,934 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=3302f0f507bd,44033,1731024872883, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T00:14:36,934 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=3302f0f507bd,44033,1731024872883, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T00:14:36,936 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,937 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,938 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,938 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=5 2024-11-08T00:14:36,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8624a8443e655fa753c4de0934012d54, server=3302f0f507bd,36139,1731024872925 in 485 msec 2024-11-08T00:14:36,942 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,943 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/bcb90ad7cb983c915be269c865806709/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8624a8443e655fa753c4de0934012d54, ASSIGN in 790 msec 2024-11-08T00:14:36,944 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened bcb90ad7cb983c915be269c865806709; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61547982, jitterRate=-0.08286359906196594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,944 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:36,944 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for bcb90ad7cb983c915be269c865806709: Running coprocessor pre-open hook at 1731024876915Writing region info on filesystem at 1731024876915Initializing all the Stores at 1731024876917 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876917Cleaning up temporary data from old regions at 1731024876927 (+10 ms)Running coprocessor post-open hooks at 1731024876944 (+17 ms)Region opened successfully at 1731024876944 2024-11-08T00:14:36,947 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/f04142b3a962ffb3cc6e9629a0aa3c6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:36,949 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,949 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened f04142b3a962ffb3cc6e9629a0aa3c6a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69621636, jitterRate=0.03744322061538696}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:36,950 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:36,950 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709., pid=45, masterSystemTime=1731024876528 2024-11-08T00:14:36,950 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for f04142b3a962ffb3cc6e9629a0aa3c6a: Running coprocessor pre-open hook at 1731024876923Writing region info on filesystem at 1731024876923Initializing all the Stores at 1731024876926 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876926Cleaning up temporary data from old regions at 1731024876938 (+12 ms)Running coprocessor post-open hooks at 1731024876950 (+12 ms)Region opened successfully at 1731024876950 2024-11-08T00:14:36,953 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a., pid=52, masterSystemTime=1731024876571 2024-11-08T00:14:36,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c8336e737af786ef3c90e95f98c755f, server=3302f0f507bd,38371,1731024872767 because future has completed 2024-11-08T00:14:36,958 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:36,958 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:36,959 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:36,959 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 66bf32aac498176881371b724c9d8008, NAME => 'test,fff,1731024875468.66bf32aac498176881371b724c9d8008.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-08T00:14:36,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3bb82385d0b72eb69ca4bdc52663bf8a, ASSIGN in 806 msec 2024-11-08T00:14:36,961 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,961 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,fff,1731024875468.66bf32aac498176881371b724c9d8008.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,961 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,961 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,962 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:36,962 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:36,962 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:36,963 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 54a0ce095f1e6524de8bdf904569509f, NAME => 'test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-08T00:14:36,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:36,964 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,964 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:36,964 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,964 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f04142b3a962ffb3cc6e9629a0aa3c6a, server=3302f0f507bd,36139,1731024872925 because future has completed 2024-11-08T00:14:36,972 INFO [StoreOpener-66bf32aac498176881371b724c9d8008-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=17 2024-11-08T00:14:36,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c8336e737af786ef3c90e95f98c755f, server=3302f0f507bd,38371,1731024872767 in 583 msec 2024-11-08T00:14:36,976 INFO [StoreOpener-66bf32aac498176881371b724c9d8008-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66bf32aac498176881371b724c9d8008 columnFamilyName family 2024-11-08T00:14:36,976 DEBUG [StoreOpener-66bf32aac498176881371b724c9d8008-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,977 INFO [StoreOpener-66bf32aac498176881371b724c9d8008-1 {}] regionserver.HStore(327): Store=66bf32aac498176881371b724c9d8008/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,977 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,978 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1c8336e737af786ef3c90e95f98c755f, ASSIGN in 823 msec 2024-11-08T00:14:36,979 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,980 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,980 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,980 INFO [StoreOpener-54a0ce095f1e6524de8bdf904569509f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,983 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:36,985 INFO [StoreOpener-54a0ce095f1e6524de8bdf904569509f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54a0ce095f1e6524de8bdf904569509f columnFamilyName family 2024-11-08T00:14:36,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=15 2024-11-08T00:14:36,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure f04142b3a962ffb3cc6e9629a0aa3c6a, server=3302f0f507bd,36139,1731024872925 in 539 msec 2024-11-08T00:14:36,985 DEBUG [StoreOpener-54a0ce095f1e6524de8bdf904569509f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:36,988 INFO [StoreOpener-54a0ce095f1e6524de8bdf904569509f-1 {}] regionserver.HStore(327): Store=54a0ce095f1e6524de8bdf904569509f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:36,988 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,990 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f04142b3a962ffb3cc6e9629a0aa3c6a, ASSIGN in 835 msec 2024-11-08T00:14:36,991 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,991 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,992 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,992 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=bcb90ad7cb983c915be269c865806709, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:36,996 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:36,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcb90ad7cb983c915be269c865806709, server=3302f0f507bd,38371,1731024872767 because future has completed 2024-11-08T00:14:37,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=27 2024-11-08T00:14:37,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,44033,1731024872883 in 669 msec 2024-11-08T00:14:37,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN in 850 msec 2024-11-08T00:14:37,009 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/66bf32aac498176881371b724c9d8008/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:37,010 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 66bf32aac498176881371b724c9d8008; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63572712, jitterRate=-0.05269277095794678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:37,011 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:37,011 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 66bf32aac498176881371b724c9d8008: Running coprocessor pre-open hook at 1731024876962Writing region info on filesystem at 1731024876962Initializing all the Stores at 1731024876966 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876966Cleaning up temporary data from old regions at 1731024876980 (+14 ms)Running coprocessor post-open hooks at 1731024877011 (+31 ms)Region opened successfully at 1731024877011 2024-11-08T00:14:37,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=26 2024-11-08T00:14:37,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure bcb90ad7cb983c915be269c865806709, server=3302f0f507bd,38371,1731024872767 in 612 msec 2024-11-08T00:14:37,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bcb90ad7cb983c915be269c865806709, ASSIGN in 862 msec 2024-11-08T00:14:37,017 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731024875468.66bf32aac498176881371b724c9d8008., pid=46, masterSystemTime=1731024876528 2024-11-08T00:14:37,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c816073c18c6a09db7883f3602eda012, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,44033,1731024872883 2024-11-08T00:14:37,023 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/54a0ce095f1e6524de8bdf904569509f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:37,023 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:37,023 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:37,023 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:37,024 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => f09fb3c8c48c1b2f01441065e691680c, NAME => 'test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-08T00:14:37,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,44033,1731024872883 because future has completed 2024-11-08T00:14:37,024 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,024 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:37,024 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,024 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,026 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=66bf32aac498176881371b724c9d8008, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:37,026 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 54a0ce095f1e6524de8bdf904569509f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75086757, jitterRate=0.11887986958026886}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:37,027 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:37,027 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 54a0ce095f1e6524de8bdf904569509f: Running coprocessor pre-open hook at 1731024876964Writing region info on filesystem at 1731024876964Initializing all the Stores at 1731024876969 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024876969Cleaning up temporary data from old regions at 1731024876992 (+23 ms)Running coprocessor post-open hooks at 1731024877027 (+35 ms)Region opened successfully at 1731024877027 2024-11-08T00:14:37,028 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f., pid=50, masterSystemTime=1731024876571 2024-11-08T00:14:37,029 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=3302f0f507bd,38371,1731024872767, table=test, region=66bf32aac498176881371b724c9d8008. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T00:14:37,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66bf32aac498176881371b724c9d8008, server=3302f0f507bd,38371,1731024872767 because future has completed 2024-11-08T00:14:37,030 INFO [StoreOpener-f09fb3c8c48c1b2f01441065e691680c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,032 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:37,032 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:37,032 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:37,032 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 335e583f3dd0eda08c4c42b80a743311, NAME => 'test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-08T00:14:37,033 INFO [StoreOpener-f09fb3c8c48c1b2f01441065e691680c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f09fb3c8c48c1b2f01441065e691680c columnFamilyName family 2024-11-08T00:14:37,033 DEBUG [StoreOpener-f09fb3c8c48c1b2f01441065e691680c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:37,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:37,033 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,033 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:37,033 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,033 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,037 INFO [StoreOpener-f09fb3c8c48c1b2f01441065e691680c-1 {}] regionserver.HStore(327): Store=f09fb3c8c48c1b2f01441065e691680c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:37,037 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54a0ce095f1e6524de8bdf904569509f, server=3302f0f507bd,36139,1731024872925 because future has completed 2024-11-08T00:14:37,039 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=3302f0f507bd,44033,1731024872883, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T00:14:37,039 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=3302f0f507bd,44033,1731024872883, table=test, region=c816073c18c6a09db7883f3602eda012. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T00:14:37,040 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,041 INFO [StoreOpener-335e583f3dd0eda08c4c42b80a743311-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,041 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=10 2024-11-08T00:14:37,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 66bf32aac498176881371b724c9d8008, server=3302f0f507bd,38371,1731024872767 in 633 msec 2024-11-08T00:14:37,042 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,042 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,044 INFO [StoreOpener-335e583f3dd0eda08c4c42b80a743311-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335e583f3dd0eda08c4c42b80a743311 columnFamilyName family 2024-11-08T00:14:37,044 DEBUG [StoreOpener-335e583f3dd0eda08c4c42b80a743311-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:37,044 INFO [StoreOpener-335e583f3dd0eda08c4c42b80a743311-1 {}] regionserver.HStore(327): Store=335e583f3dd0eda08c4c42b80a743311/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:37,045 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=66bf32aac498176881371b724c9d8008, ASSIGN in 891 msec 2024-11-08T00:14:37,046 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,046 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,048 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,048 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,048 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,050 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/f09fb3c8c48c1b2f01441065e691680c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:37,051 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,051 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened f09fb3c8c48c1b2f01441065e691680c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59697910, jitterRate=-0.11043182015419006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:37,051 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:37,051 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for f09fb3c8c48c1b2f01441065e691680c: Running coprocessor pre-open hook at 1731024877024Writing region info on filesystem at 1731024877024Initializing all the Stores at 1731024877026 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024877026Cleaning up temporary data from old regions at 1731024877042 (+16 ms)Running coprocessor post-open hooks at 1731024877051 (+9 ms)Region opened successfully at 1731024877051 2024-11-08T00:14:37,052 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c., pid=39, masterSystemTime=1731024876528 2024-11-08T00:14:37,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=25 2024-11-08T00:14:37,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 54a0ce095f1e6524de8bdf904569509f, server=3302f0f507bd,36139,1731024872925 in 619 msec 2024-11-08T00:14:37,056 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:37,056 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:37,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=54a0ce095f1e6524de8bdf904569509f, ASSIGN in 902 msec 2024-11-08T00:14:37,058 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:37,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f09fb3c8c48c1b2f01441065e691680c, server=3302f0f507bd,38371,1731024872767 because future has completed 2024-11-08T00:14:37,063 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/335e583f3dd0eda08c4c42b80a743311/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:14:37,064 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 335e583f3dd0eda08c4c42b80a743311; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58970397, jitterRate=-0.12127260863780975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:37,064 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:37,064 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 335e583f3dd0eda08c4c42b80a743311: Running coprocessor pre-open hook at 1731024877034Writing region info on filesystem at 1731024877034Initializing all the Stores at 1731024877038 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024877039 (+1 ms)Cleaning up temporary data from old regions at 1731024877048 (+9 ms)Running coprocessor post-open hooks at 1731024877064 (+16 ms)Region opened successfully at 1731024877064 2024-11-08T00:14:37,066 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311., pid=51, masterSystemTime=1731024876571 2024-11-08T00:14:37,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-08T00:14:37,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,44033,1731024872883 in 741 msec 2024-11-08T00:14:37,068 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:37,069 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:37,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN in 916 msec 2024-11-08T00:14:37,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:37,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335e583f3dd0eda08c4c42b80a743311, server=3302f0f507bd,36139,1731024872925 because future has completed 2024-11-08T00:14:37,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=29 2024-11-08T00:14:37,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure f09fb3c8c48c1b2f01441065e691680c, server=3302f0f507bd,38371,1731024872767 in 710 msec 2024-11-08T00:14:37,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f09fb3c8c48c1b2f01441065e691680c, ASSIGN in 927 msec 2024-11-08T00:14:37,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=28 2024-11-08T00:14:37,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 335e583f3dd0eda08c4c42b80a743311, server=3302f0f507bd,36139,1731024872925 in 649 msec 2024-11-08T00:14:37,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=335e583f3dd0eda08c4c42b80a743311, ASSIGN in 934 msec 2024-11-08T00:14:37,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=11 2024-11-08T00:14:37,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,44033,1731024872883 in 751 msec 2024-11-08T00:14:37,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=4 2024-11-08T00:14:37,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN in 943 msec 2024-11-08T00:14:37,111 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T00:14:37,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731024877111"}]},"ts":"1731024877111"} 2024-11-08T00:14:37,114 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-08T00:14:37,116 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T00:14:37,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.6380 sec 2024-11-08T00:14:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:14:37,668 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-08T00:14:37,668 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-08T00:14:37,669 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:14:37,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-08T00:14:37,688 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:14:37,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-08T00:14:37,689 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:14:37,695 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-08T00:14:37,723 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-08T00:14:37,752 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test', row='', locateType=CURRENT is [region=test,,1731024875468.8624a8443e655fa753c4de0934012d54., hostname=3302f0f507bd,36139,1731024872925, seqNum=2] 2024-11-08T00:14:37,755 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:14:37,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:14:37,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:14:37,795 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:14:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8624a8443e655fa753c4de0934012d54 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ffaf29acbc927f398077e41ad232c549 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cd2627434f69d79290b6a35838b5f326 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b5026dfa3bad2942533f117d619b1656 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f04142b3a962ffb3cc6e9629a0aa3c6a changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3bddcc13c4eccc1d32b3c09b1dcb1583 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b8e8ac9592d4e973f9896a6807e10272 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 54a0ce095f1e6524de8bdf904569509f changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 335e583f3dd0eda08c4c42b80a743311 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9c5dafc4711af09b862954533a7784ca changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 66bf32aac498176881371b724c9d8008 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 13f20a01a04306fecc9b545706c9d14c changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3bb82385d0b72eb69ca4bdc52663bf8a changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1c8336e737af786ef3c90e95f98c755f changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 850fbeeee9e2784d3676499ac7582114 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2e10c5ec7fdc210db94ac8df9187f1c4 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bcb90ad7cb983c915be269c865806709 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f09fb3c8c48c1b2f01441065e691680c changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a594c54df520141421a62d3f056e15f2 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region faad0b2f1a2327cef2cd0aefa9a4d1c8 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2ef74f869fdb87a20df612d1e6e2c262 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ee421baf774947d334ec46882ae42d9c changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7acb38c712f4cad61cf5f78f7ffa9a18 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8280a310195422b4e0184bc7c709a2e9 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 40e3fed5cf6cc7497a1c2220e1de9b9b changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c816073c18c6a09db7883f3602eda012 changed from -1.0 to 0.0, refreshing cache 2024-11-08T00:14:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-08T00:14:37,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-08T00:14:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-08T00:14:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(204): Hosts are {3302f0f507bd=0} racks are {/default-rack=0} 2024-11-08T00:14:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-08T00:14:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-08T00:14:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-08T00:14:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T00:14:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T00:14:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T00:14:37,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T00:14:37,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T00:14:37,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T00:14:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-08T00:14:37,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-08T00:14:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-08T00:14:37,946 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:14:37,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-08T00:14:38,009 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=373, OpenFileDescriptor=605, MaxFileDescriptor=1048576, SystemLoadAverage=926, ProcessCount=11, AvailableMemoryMB=1176 2024-11-08T00:14:38,013 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-08T00:14:38,014 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-08T00:14:38,014 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 3302f0f507bd,44033,1731024872883 2024-11-08T00:14:38,015 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 3302f0f507bd,44033,1731024872883: Simulated kill ***** 2024-11-08T00:14:38,016 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-08T00:14:38,020 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-08T00:14:38,024 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-08T00:14:38,024 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-08T00:14:38,024 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-08T00:14:38,025 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 292685552 }, "NonHeapMemoryUsage": { "committed": 151519232, "init": 7667712, "max": -1, "used": 149202840 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "3302f0f507bd" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "3302f0f507bd" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "3302f0f507bd" } ] } 2024-11-08T00:14:38,028 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36429 {}] master.MasterRpcServices(700): 3302f0f507bd,44033,1731024872883 reported a fatal error: ***** ABORTING region server 3302f0f507bd,44033,1731024872883: Simulated kill ***** 2024-11-08T00:14:38,031 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3302f0f507bd,44033,1731024872883' ***** 2024-11-08T00:14:38,031 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-08T00:14:38,032 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T00:14:38,033 INFO [RS:1;3302f0f507bd:44033 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-08T00:14:38,033 INFO [RS:1;3302f0f507bd:44033 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-08T00:14:38,033 INFO [RS:1;3302f0f507bd:44033 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:14:38,033 INFO [RS:1;3302f0f507bd:44033 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3302f0f507bd:44033. 2024-11-08T00:14:38,034 DEBUG [RS:1;3302f0f507bd:44033 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:14:38,037 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T00:14:38,042 DEBUG [RS:1;3302f0f507bd:44033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:38,045 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:14:38,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741836_1012 (size=32114) 2024-11-08T00:14:38,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741836_1012 (size=32114) 2024-11-08T00:14:38,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741836_1012 (size=32114) 2024-11-08T00:14:38,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741833_1009 (size=2237) 2024-11-08T00:14:38,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741833_1009 (size=2237) 2024-11-08T00:14:38,096 DEBUG [RS:1;3302f0f507bd:44033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:38,097 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:14:38,098 INFO [RS:1;3302f0f507bd:44033 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44033 2024-11-08T00:14:38,108 INFO [regionserver/3302f0f507bd:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:14:38,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:14:38,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3302f0f507bd,44033,1731024872883 2024-11-08T00:14:38,117 INFO [RS:1;3302f0f507bd:44033 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:14:38,119 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3302f0f507bd,44033,1731024872883] 2024-11-08T00:14:38,151 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3302f0f507bd,44033,1731024872883 already deleted, retry=false 2024-11-08T00:14:38,151 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 3302f0f507bd,44033,1731024872883 on 3302f0f507bd,36429,1731024871043 2024-11-08T00:14:38,158 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:38,162 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=57 for 3302f0f507bd,44033,1731024872883 (carryingMeta=true) 3302f0f507bd,44033,1731024872883/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@59c92850[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T00:14:38,162 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:38,165 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true, isMeta: true 2024-11-08T00:14:38,167 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting 2024-11-08T00:14:38,169 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3302f0f507bd,44033,1731024872883 WAL count=1, meta=true 2024-11-08T00:14:38,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta}] 2024-11-08T00:14:38,181 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:38,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:38,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:14:38,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44033-0x10117cb57f40002, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:14:38,231 INFO [RS:1;3302f0f507bd:44033 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:14:38,231 INFO [RS:1;3302f0f507bd:44033 {}] regionserver.HRegionServer(1031): Exiting; stopping=3302f0f507bd,44033,1731024872883; zookeeper connection closed. 2024-11-08T00:14:38,231 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a0b2058 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a0b2058 2024-11-08T00:14:38,247 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T00:14:38,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-08T00:14:38,385 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, size=31.4 K (32114bytes) 2024-11-08T00:14:38,386 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-08T00:14:38,386 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-08T00:14:38,386 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta 2024-11-08T00:14:38,395 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta after 6ms 2024-11-08T00:14:38,405 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta took 22ms 2024-11-08T00:14:38,417 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-08T00:14:38,426 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta.temp 2024-11-08T00:14:38,430 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta so closing down 2024-11-08T00:14:38,430 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-08T00:14:38,431 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta.temp 2024-11-08T00:14:38,434 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-08T00:14:38,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741864_1040 (size=31873) 2024-11-08T00:14:38,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741864_1040 (size=31873) 2024-11-08T00:14:38,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741864_1040 (size=31873) 2024-11-08T00:14:38,450 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-08T00:14:38,453 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta.temp to hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T00:14:38,456 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 46 ms; skipped=1; WAL=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-08T00:14:38,456 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, journal: Splitting hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, size=31.4 K (32114bytes) at 1731024878385Finishing writing output for hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta so closing down at 1731024878430 (+45 ms)Creating recovered edits writer path=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta.temp at 1731024878431 (+1 ms)3 split writer threads finished at 1731024878434 (+3 ms)Closed recovered edits writer path=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1731024878450 (+16 ms)Rename recovered edits hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta.temp to hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731024878453 (+3 ms)Processed 58 edits across 1 Regions in 46 ms; skipped=1; WAL=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731024878456 (+3 ms) 2024-11-08T00:14:38,457 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta 2024-11-08T00:14:38,458 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-08T00:14:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-08T00:14:38,468 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta to hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs 2024-11-08T00:14:38,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T00:14:38,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 in 283 msec 2024-11-08T00:14:38,473 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:38,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T00:14:38,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 in 303 msec 2024-11-08T00:14:38,479 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting dir is empty, no logs to split. 2024-11-08T00:14:38,479 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3302f0f507bd,44033,1731024872883 WAL count=0, meta=true 2024-11-08T00:14:38,480 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3302f0f507bd,44033,1731024872883 WAL splitting is done? wals=0, meta=true 2024-11-08T00:14:38,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T00:14:38,484 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:38,486 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:14:38,502 INFO [regionserver/3302f0f507bd:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T00:14:38,502 INFO [regionserver/3302f0f507bd:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T00:14:38,636 DEBUG [3302f0f507bd:36429 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T00:14:38,636 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(204): Hosts are {3302f0f507bd=0} racks are {/default-rack=0} 2024-11-08T00:14:38,637 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T00:14:38,637 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T00:14:38,637 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T00:14:38,637 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T00:14:38,637 INFO [3302f0f507bd:36429 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T00:14:38,637 INFO [3302f0f507bd:36429 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T00:14:38,637 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T00:14:38,638 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:38,640 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPENING 2024-11-08T00:14:38,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:38,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:38,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:38,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:38,657 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:38,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:38,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:38,659 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:38,823 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T00:14:38,824 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:14:38,824 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T00:14:38,831 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3302f0f507bd%2C36139%2C1731024872925.meta, suffix=.meta, logDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,36139,1731024872925, archiveDir=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs, maxLogs=32 2024-11-08T00:14:38,854 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,36139,1731024872925/3302f0f507bd%2C36139%2C1731024872925.meta.1731024878833.meta, exclude list is [], retry=0 2024-11-08T00:14:38,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36957,DS-05680217-d0db-4d7d-a701-2d3460612161,DISK] 2024-11-08T00:14:38,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36391,DS-c7c86592-a2a6-4354-ab71-9748dcc4c027,DISK] 2024-11-08T00:14:38,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44189,DS-697ab831-8e11-49a9-ae3f-5d4889ea6c9b,DISK] 2024-11-08T00:14:38,873 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,36139,1731024872925/3302f0f507bd%2C36139%2C1731024872925.meta.1731024878833.meta 2024-11-08T00:14:38,876 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39279:39279),(127.0.0.1/127.0.0.1:43187:43187),(127.0.0.1/127.0.0.1:44713:44713)] 2024-11-08T00:14:38,877 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:14:38,877 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T00:14:38,877 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T00:14:38,878 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T00:14:38,878 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T00:14:38,878 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:38,878 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T00:14:38,878 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T00:14:38,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T00:14:38,885 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T00:14:38,885 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:38,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:38,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T00:14:38,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T00:14:38,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:38,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:38,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T00:14:38,890 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T00:14:38,890 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:38,891 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:38,891 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T00:14:38,892 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T00:14:38,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:38,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:14:38,893 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T00:14:38,894 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740 2024-11-08T00:14:38,896 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740 2024-11-08T00:14:38,897 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T00:14:38,912 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T00:14:38,915 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-08T00:14:38,981 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/info/7aa857dec4b64a5d97866ea30a88228e is 138, key is test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2./info:regioninfo/1731024876904/Put/seqid=0 2024-11-08T00:14:38,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741866_1042 (size=34924) 2024-11-08T00:14:38,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741866_1042 (size=34924) 2024-11-08T00:14:38,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741866_1042 (size=34924) 2024-11-08T00:14:38,999 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/info/7aa857dec4b64a5d97866ea30a88228e 2024-11-08T00:14:39,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741829_1005 (size=34) 2024-11-08T00:14:39,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741838_1014 (size=42) 2024-11-08T00:14:39,084 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/ns/8feba4dbafe64b399e5328177d848b6b is 43, key is default/ns:d/1731024875236/Put/seqid=0 2024-11-08T00:14:39,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741867_1043 (size=5153) 2024-11-08T00:14:39,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741867_1043 (size=5153) 2024-11-08T00:14:39,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741867_1043 (size=5153) 2024-11-08T00:14:39,097 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/ns/8feba4dbafe64b399e5328177d848b6b 2024-11-08T00:14:39,129 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/table/f84eb75de49241c4966fdbfa3bffa65a is 40, key is test/table:state/1731024877111/Put/seqid=0 2024-11-08T00:14:39,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741868_1044 (size=5165) 2024-11-08T00:14:39,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741868_1044 (size=5165) 2024-11-08T00:14:39,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741868_1044 (size=5165) 2024-11-08T00:14:39,150 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/table/f84eb75de49241c4966fdbfa3bffa65a 2024-11-08T00:14:39,169 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/info/7aa857dec4b64a5d97866ea30a88228e as hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/info/7aa857dec4b64a5d97866ea30a88228e 2024-11-08T00:14:39,181 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/info/7aa857dec4b64a5d97866ea30a88228e, entries=260, sequenceid=60, filesize=34.1 K 2024-11-08T00:14:39,184 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/ns/8feba4dbafe64b399e5328177d848b6b as hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/ns/8feba4dbafe64b399e5328177d848b6b 2024-11-08T00:14:39,197 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/ns/8feba4dbafe64b399e5328177d848b6b, entries=2, sequenceid=60, filesize=5.0 K 2024-11-08T00:14:39,199 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/table/f84eb75de49241c4966fdbfa3bffa65a as hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/table/f84eb75de49241c4966fdbfa3bffa65a 2024-11-08T00:14:39,222 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/table/f84eb75de49241c4966fdbfa3bffa65a, entries=2, sequenceid=60, filesize=5.0 K 2024-11-08T00:14:39,228 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 310ms, sequenceid=60, compaction requested=false; wal=null 2024-11-08T00:14:39,229 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T00:14:39,233 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T00:14:39,238 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T00:14:39,238 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T00:14:39,240 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:14:39,248 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T00:14:39,251 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-08T00:14:39,253 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70217579, jitterRate=0.04632346332073212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:14:39,253 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T00:14:39,254 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731024878878Writing region info on filesystem at 1731024878878Initializing all the Stores at 1731024878880 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024878880Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024878883 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024878883Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731024878883Obtaining lock to block concurrent updates at 1731024878915 (+32 ms)Preparing flush snapshotting stores in 1588230740 at 1731024878915Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731024878919 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731024878919Flushing 1588230740/info: creating writer at 1731024878920 (+1 ms)Flushing 1588230740/info: appending metadata at 1731024878969 (+49 ms)Flushing 1588230740/info: closing flushed file at 1731024878972 (+3 ms)Flushing 1588230740/ns: creating writer at 1731024879060 (+88 ms)Flushing 1588230740/ns: appending metadata at 1731024879084 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1731024879084Flushing 1588230740/table: creating writer at 1731024879111 (+27 ms)Flushing 1588230740/table: appending metadata at 1731024879128 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731024879128Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cb5b182: reopening flushed file at 1731024879167 (+39 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d7dd069: reopening flushed file at 1731024879183 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@329266bc: reopening flushed file at 1731024879197 (+14 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 310ms, sequenceid=60, compaction requested=false; wal=null at 1731024879228 (+31 ms)Cleaning up temporary data from old regions at 1731024879238 (+10 ms)Running coprocessor post-open hooks at 1731024879253 (+15 ms)Region opened successfully at 1731024879253 2024-11-08T00:14:39,257 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731024878811 2024-11-08T00:14:39,260 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T00:14:39,260 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T00:14:39,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,261 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:39,265 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T00:14:39,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:39,277 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:39,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:39,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:39,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:39,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:39,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T00:14:39,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 in 620 msec 2024-11-08T00:14:39,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-08T00:14:39,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 804 msec 2024-11-08T00:14:39,290 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 3302f0f507bd,44033,1731024872883 had 9 regions 2024-11-08T00:14:39,291 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true, isMeta: false 2024-11-08T00:14:39,293 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3302f0f507bd,44033,1731024872883 WAL count=1, meta=false 2024-11-08T00:14:39,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653}] 2024-11-08T00:14:39,296 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:39,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741828_1004 (size=1189) 2024-11-08T00:14:39,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741839_1015 (size=45) 2024-11-08T00:14:39,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-08T00:14:39,471 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653, size=2.2 K (2237bytes) 2024-11-08T00:14:39,471 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653 2024-11-08T00:14:39,471 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653 after 0ms 2024-11-08T00:14:39,474 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653 took 4ms 2024-11-08T00:14:39,478 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2ef74f869fdb87a20df612d1e6e2c262: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T00:14:39,480 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8280a310195422b4e0184bc7c709a2e9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T00:14:39,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7acb38c712f4cad61cf5f78f7ffa9a18: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T00:14:39,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ee421baf774947d334ec46882ae42d9c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T00:14:39,486 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for faad0b2f1a2327cef2cd0aefa9a4d1c8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T00:14:39,488 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c816073c18c6a09db7883f3602eda012: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T00:14:39,490 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 40e3fed5cf6cc7497a1c2220e1de9b9b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T00:14:39,492 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a594c54df520141421a62d3f056e15f2: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-08T00:14:39,492 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653 so closing down 2024-11-08T00:14:39,492 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-08T00:14:39,493 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-08T00:14:39,493 INFO [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-08T00:14:39,493 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653, journal: Splitting hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653, size=2.2 K (2237bytes) at 1731024879471Finishing writing output for hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653 so closing down at 1731024879492 (+21 ms)3 split writer threads finished at 1731024879493 (+1 ms)Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1731024879493 2024-11-08T00:14:39,493 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653 2024-11-08T00:14:39,493 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-08T00:14:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36429 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-08T00:14:39,498 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting/3302f0f507bd%2C44033%2C1731024872883.1731024874653 to hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs 2024-11-08T00:14:39,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-08T00:14:39,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 in 201 msec 2024-11-08T00:14:39,503 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-08T00:14:39,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 in 210 msec 2024-11-08T00:14:39,508 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/WALs/3302f0f507bd,44033,1731024872883-splitting dir is empty, no logs to split. 2024-11-08T00:14:39,508 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3302f0f507bd,44033,1731024872883 WAL count=0, meta=false 2024-11-08T00:14:39,508 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3302f0f507bd,44033,1731024872883 WAL splitting is done? wals=0, meta=false 2024-11-08T00:14:39,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN}] 2024-11-08T00:14:39,512 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:39,513 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:14:39,514 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:39,515 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:14:39,516 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:39,517 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:14:39,518 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:39,519 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:14:39,519 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:39,520 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:14:39,521 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:39,522 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:14:39,524 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:39,526 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:14:39,527 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:39,528 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T00:14:39,664 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(204): Hosts are {3302f0f507bd=0} racks are {/default-rack=0} 2024-11-08T00:14:39,664 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T00:14:39,664 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T00:14:39,664 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T00:14:39,664 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T00:14:39,664 INFO [3302f0f507bd:36429 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T00:14:39,664 INFO [3302f0f507bd:36429 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T00:14:39,664 DEBUG [3302f0f507bd:36429 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T00:14:39,665 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c816073c18c6a09db7883f3602eda012, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,666 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,667 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,667 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3302f0f507bd:44033 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:14:39,668 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, regionLocation=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,668 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1, error=java.net.ConnectException: Call to address=3302f0f507bd:44033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 2024-11-08T00:14:39,669 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,669 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T00:14:39,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1 from cache 2024-11-08T00:14:39,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=a594c54df520141421a62d3f056e15f2, regionState=OPENING, regionLocation=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,670 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:14:39,670 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,36139,1731024872925, seqNum=-1] 2024-11-08T00:14:39,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 3302f0f507bd:44033 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 2024-11-08T00:14:39,671 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:14:39,671 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3302f0f507bd:44033 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:14:39,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,36139,1731024872925, seqNum=-1, error=java.net.ConnectException: Call to address=3302f0f507bd:44033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 2024-11-08T00:14:39,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,36139,1731024872925, seqNum=-1, error=java.net.ConnectException: Call to address=3302f0f507bd:44033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 2024-11-08T00:14:39,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,36139,1731024872925, seqNum=-1, error=java.net.ConnectException: Call to address=3302f0f507bd:44033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 2024-11-08T00:14:39,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,36139,1731024872925, seqNum=-1, error=java.net.ConnectException: Call to address=3302f0f507bd:44033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 2024-11-08T00:14:39,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,36139,1731024872925, seqNum=-1, error=java.net.ConnectException: Call to address=3302f0f507bd:44033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 2024-11-08T00:14:39,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,44033,1731024872883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,36139,1731024872925, seqNum=-1, error=java.net.ConnectException: Call to address=3302f0f507bd:44033 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 2024-11-08T00:14:39,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 3302f0f507bd:44033 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3302f0f507bd/172.17.0.3:44033 2024-11-08T00:14:39,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51521, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:14:39,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN because future has completed 2024-11-08T00:14:39,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:39,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN because future has completed 2024-11-08T00:14:39,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:39,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN because future has completed 2024-11-08T00:14:39,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:39,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN because future has completed 2024-11-08T00:14:39,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN because future has completed 2024-11-08T00:14:39,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN because future has completed 2024-11-08T00:14:39,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN because future has completed 2024-11-08T00:14:39,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN because future has completed 2024-11-08T00:14:39,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:39,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:39,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,38371,1731024872767}] 2024-11-08T00:14:39,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:39,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee421baf774947d334ec46882ae42d9c, server=3302f0f507bd,36139,1731024872925}] 2024-11-08T00:14:39,840 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:39,840 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => a594c54df520141421a62d3f056e15f2, NAME => 'test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-08T00:14:39,840 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,840 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:39,841 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,841 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,843 INFO [StoreOpener-a594c54df520141421a62d3f056e15f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,845 INFO [StoreOpener-a594c54df520141421a62d3f056e15f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a594c54df520141421a62d3f056e15f2 columnFamilyName family 2024-11-08T00:14:39,845 DEBUG [StoreOpener-a594c54df520141421a62d3f056e15f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:39,845 INFO [StoreOpener-a594c54df520141421a62d3f056e15f2-1 {}] regionserver.HStore(327): Store=a594c54df520141421a62d3f056e15f2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:39,845 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,846 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,848 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,849 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,849 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,852 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,854 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened a594c54df520141421a62d3f056e15f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65377680, jitterRate=-0.02579665184020996}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:39,854 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:39,854 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for a594c54df520141421a62d3f056e15f2: Running coprocessor pre-open hook at 1731024879841Writing region info on filesystem at 1731024879841Initializing all the Stores at 1731024879843 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024879843Cleaning up temporary data from old regions at 1731024879849 (+6 ms)Running coprocessor post-open hooks at 1731024879854 (+5 ms)Region opened successfully at 1731024879854 2024-11-08T00:14:39,856 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2., pid=72, masterSystemTime=1731024879833 2024-11-08T00:14:39,859 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:39,859 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:39,860 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:39,860 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ef74f869fdb87a20df612d1e6e2c262, NAME => 'test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-08T00:14:39,860 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,861 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:39,861 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,861 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,864 INFO [StoreOpener-2ef74f869fdb87a20df612d1e6e2c262-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,866 INFO [StoreOpener-2ef74f869fdb87a20df612d1e6e2c262-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ef74f869fdb87a20df612d1e6e2c262 columnFamilyName family 2024-11-08T00:14:39,866 DEBUG [StoreOpener-2ef74f869fdb87a20df612d1e6e2c262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:39,867 INFO [StoreOpener-2ef74f869fdb87a20df612d1e6e2c262-1 {}] regionserver.HStore(327): Store=2ef74f869fdb87a20df612d1e6e2c262/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:39,867 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,869 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-08T00:14:39,869 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,869 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,869 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:39,873 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,874 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,874 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,877 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,878 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 2ef74f869fdb87a20df612d1e6e2c262; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69277536, jitterRate=0.032315731048583984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:39,879 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:39,879 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 2ef74f869fdb87a20df612d1e6e2c262: Running coprocessor pre-open hook at 1731024879861Writing region info on filesystem at 1731024879861Initializing all the Stores at 1731024879863 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024879863Cleaning up temporary data from old regions at 1731024879874 (+11 ms)Running coprocessor post-open hooks at 1731024879879 (+5 ms)Region opened successfully at 1731024879879 2024-11-08T00:14:39,880 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262., pid=75, masterSystemTime=1731024879833 2024-11-08T00:14:39,882 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:39,882 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:39,883 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:39,881 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-08T00:14:39,883 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 40e3fed5cf6cc7497a1c2220e1de9b9b, NAME => 'test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-08T00:14:39,883 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,883 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:39,883 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,883 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,883 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:39,883 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee421baf774947d334ec46882ae42d9c, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:39,884 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee421baf774947d334ec46882ae42d9c, server=3302f0f507bd,36139,1731024872925 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:39,885 INFO [StoreOpener-40e3fed5cf6cc7497a1c2220e1de9b9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,887 INFO [StoreOpener-40e3fed5cf6cc7497a1c2220e1de9b9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40e3fed5cf6cc7497a1c2220e1de9b9b columnFamilyName family 2024-11-08T00:14:39,887 DEBUG [StoreOpener-40e3fed5cf6cc7497a1c2220e1de9b9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:39,888 INFO [StoreOpener-40e3fed5cf6cc7497a1c2220e1de9b9b-1 {}] regionserver.HStore(327): Store=40e3fed5cf6cc7497a1c2220e1de9b9b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:39,888 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,889 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,891 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,892 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,892 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,894 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,895 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 40e3fed5cf6cc7497a1c2220e1de9b9b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72898655, jitterRate=0.08627460896968842}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:39,895 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:39,896 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 40e3fed5cf6cc7497a1c2220e1de9b9b: Running coprocessor pre-open hook at 1731024879883Writing region info on filesystem at 1731024879883Initializing all the Stores at 1731024879885 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024879885Cleaning up temporary data from old regions at 1731024879892 (+7 ms)Running coprocessor post-open hooks at 1731024879895 (+3 ms)Region opened successfully at 1731024879896 (+1 ms) 2024-11-08T00:14:39,896 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b., pid=77, masterSystemTime=1731024879833 2024-11-08T00:14:39,900 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:39,900 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:39,900 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:39,900 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 7acb38c712f4cad61cf5f78f7ffa9a18, NAME => 'test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-08T00:14:39,901 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,901 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:39,901 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,901 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,903 INFO [StoreOpener-7acb38c712f4cad61cf5f78f7ffa9a18-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,904 INFO [StoreOpener-7acb38c712f4cad61cf5f78f7ffa9a18-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7acb38c712f4cad61cf5f78f7ffa9a18 columnFamilyName family 2024-11-08T00:14:39,904 DEBUG [StoreOpener-7acb38c712f4cad61cf5f78f7ffa9a18-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:39,905 INFO [StoreOpener-7acb38c712f4cad61cf5f78f7ffa9a18-1 {}] regionserver.HStore(327): Store=7acb38c712f4cad61cf5f78f7ffa9a18/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:39,905 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,906 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,908 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,908 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,909 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,911 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,913 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 7acb38c712f4cad61cf5f78f7ffa9a18; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67678005, jitterRate=0.008480861783027649}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:39,913 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:39,913 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 7acb38c712f4cad61cf5f78f7ffa9a18: Running coprocessor pre-open hook at 1731024879901Writing region info on filesystem at 1731024879901Initializing all the Stores at 1731024879903 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024879903Cleaning up temporary data from old regions at 1731024879909 (+6 ms)Running coprocessor post-open hooks at 1731024879913 (+4 ms)Region opened successfully at 1731024879913 2024-11-08T00:14:39,914 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18., pid=76, masterSystemTime=1731024879833 2024-11-08T00:14:39,916 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:39,916 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:39,940 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. 2024-11-08T00:14:39,940 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 8280a310195422b4e0184bc7c709a2e9, NAME => 'test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-08T00:14:39,940 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,940 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:14:39,940 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,941 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,943 INFO [StoreOpener-8280a310195422b4e0184bc7c709a2e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,944 INFO [StoreOpener-8280a310195422b4e0184bc7c709a2e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8280a310195422b4e0184bc7c709a2e9 columnFamilyName family 2024-11-08T00:14:39,944 DEBUG [StoreOpener-8280a310195422b4e0184bc7c709a2e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:14:39,946 INFO [StoreOpener-8280a310195422b4e0184bc7c709a2e9-1 {}] regionserver.HStore(327): Store=8280a310195422b4e0184bc7c709a2e9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:14:39,947 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,948 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,950 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,951 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:39,951 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:39,951 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,951 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,951 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:39,952 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:39,953 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:39,953 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:39,953 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:39,953 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:39,954 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,956 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-08T00:14:39,956 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 8280a310195422b4e0184bc7c709a2e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73959236, jitterRate=0.10207849740982056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T00:14:39,956 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8280a310195422b4e0184bc7c709a2e9 2024-11-08T00:14:39,956 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 8280a310195422b4e0184bc7c709a2e9: Running coprocessor pre-open hook at 1731024879941Writing region info on filesystem at 1731024879941Initializing all the Stores at 1731024879942 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731024879942Cleaning up temporary data from old regions at 1731024879951 (+9 ms)Running coprocessor post-open hooks at 1731024879956 (+5 ms)Region opened successfully at 1731024879956 2024-11-08T00:14:39,961 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9., pid=74, masterSystemTime=1731024879934 2024-11-08T00:14:39,962 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:39,963 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:39,964 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:39,965 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:39,966 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:39,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:39,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:39,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee421baf774947d334ec46882ae42d9c, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,980 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:39,981 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:39,981 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:39,981 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:39,981 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:39,981 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:39,981 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:39,981 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:39,981 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,981 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,982 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,982 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,982 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee421baf774947d334ec46882ae42d9c, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:39,983 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 27 msec 2024-11-08T00:14:39,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:39,986 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:39,986 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:39,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:39,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:39,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:39,993 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:39,993 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:39,993 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:39,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:39,996 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:39,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:39,997 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:39,997 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:39,997 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:39,997 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:39,997 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:39,997 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee421baf774947d334ec46882ae42d9c to restore RIT 2024-11-08T00:14:39,997 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:39,997 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "3302f0f507bd" port: 36139 start_code: 1731024872925 } transition { transition_code: OPENED region_info { region_id: 1731024875468 table_name { namespace: "default" qualifier: "test" } start_key: "ttt" end_key: "uuu" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 74 initiating_master_active_time: 1731024873078 }; retry (#0) after 1006ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T00:14:39,999 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024909999 2024-11-08T00:14:39,999 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:39,999 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,001 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,002 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,002 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,002 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,36139,1731024872925 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,047 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,047 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940047 2024-11-08T00:14:40,047 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000047 2024-11-08T00:14:40,047 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 48 msec 2024-11-08T00:14:40,047 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,047 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,047 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,047 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,048 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,048 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,048 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,048 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,055 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,067 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure faad0b2f1a2327cef2cd0aefa9a4d1c8, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,068 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 19 msec 2024-11-08T00:14:40,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,070 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,071 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,078 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,078 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,079 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,080 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,080 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,080 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,080 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,080 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,080 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:40,080 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee421baf774947d334ec46882ae42d9c to restore RIT 2024-11-08T00:14:40,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910084 2024-11-08T00:14:40,084 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,084 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,087 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,092 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,092 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,092 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,38371,1731024872767 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,105 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,105 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940105 2024-11-08T00:14:40,106 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000105 2024-11-08T00:14:40,106 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 21 msec 2024-11-08T00:14:40,106 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,106 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,106 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,106 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,107 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,107 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,107 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,107 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,108 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,121 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,122 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e3fed5cf6cc7497a1c2220e1de9b9b, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,123 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-08T00:14:40,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,124 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,124 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,137 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,137 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,137 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,137 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,137 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,137 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,138 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:40,138 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee421baf774947d334ec46882ae42d9c to restore RIT 2024-11-08T00:14:40,145 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910145 2024-11-08T00:14:40,145 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,145 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,147 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,148 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,148 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,148 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,158 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,158 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940158 2024-11-08T00:14:40,158 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000158 2024-11-08T00:14:40,158 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-08T00:14:40,158 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,159 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,159 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,159 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,159 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,159 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,159 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,163 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,164 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,170 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7acb38c712f4cad61cf5f78f7ffa9a18, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,170 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-08T00:14:40,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,171 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,171 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,183 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,183 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,183 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,183 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,183 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,183 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,184 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,184 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,184 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,184 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:40,184 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee421baf774947d334ec46882ae42d9c to restore RIT 2024-11-08T00:14:40,184 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910184 2024-11-08T00:14:40,184 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,184 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,185 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,186 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,186 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,186 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,198 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,198 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940198 2024-11-08T00:14:40,198 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000198 2024-11-08T00:14:40,198 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-08T00:14:40,198 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,199 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,199 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,199 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,199 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,199 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,199 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,200 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,204 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,204 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,211 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ef74f869fdb87a20df612d1e6e2c262, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,212 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-08T00:14:40,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,214 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,214 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,225 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,226 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,226 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,226 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:40,226 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee421baf774947d334ec46882ae42d9c to restore RIT 2024-11-08T00:14:40,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910226 2024-11-08T00:14:40,226 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,226 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,227 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,229 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,229 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,229 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,240 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,240 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940240 2024-11-08T00:14:40,240 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000240 2024-11-08T00:14:40,240 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-08T00:14:40,240 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,240 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,240 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,240 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,240 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,240 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,240 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,241 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,242 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,244 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,245 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,246 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,248 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,249 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,250 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,250 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8280a310195422b4e0184bc7c709a2e9, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-08T00:14:40,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,252 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,252 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:40,265 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee421baf774947d334ec46882ae42d9c to restore RIT 2024-11-08T00:14:40,266 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910266 2024-11-08T00:14:40,266 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,266 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,267 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,268 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,269 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,269 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,277 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,277 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940277 2024-11-08T00:14:40,277 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000277 2024-11-08T00:14:40,277 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-08T00:14:40,277 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,277 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,277 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,277 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,278 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,278 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,278 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,278 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,282 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,288 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c816073c18c6a09db7883f3602eda012, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,288 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-08T00:14:40,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,289 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,289 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,298 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,298 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,298 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:40,300 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee421baf774947d334ec46882ae42d9c to restore RIT 2024-11-08T00:14:40,301 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910301 2024-11-08T00:14:40,301 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,301 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,302 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,303 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,303 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,304 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,315 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,315 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940315 2024-11-08T00:14:40,315 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000315 2024-11-08T00:14:40,316 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-08T00:14:40,316 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,316 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,316 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,316 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,316 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,316 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,316 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,316 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,317 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,320 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,324 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,325 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,325 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,325 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-08T00:14:40,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,326 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,326 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:40,339 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee421baf774947d334ec46882ae42d9c to restore RIT 2024-11-08T00:14:40,339 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910339 2024-11-08T00:14:40,340 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,340 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,341 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,342 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,342 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,342 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,346 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,347 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,349 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,349 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940349 2024-11-08T00:14:40,349 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000349 2024-11-08T00:14:40,349 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T00:14:40,350 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,350 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,350 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,350 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,350 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,350 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,350 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,351 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,351 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,354 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,355 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,358 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,359 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,359 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN 2024-11-08T00:14:40,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-08T00:14:40,360 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,360 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,360 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,376 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,376 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:40,377 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee421baf774947d334ec46882ae42d9c, ASSIGN to state=OFFLINE, location=null, table=test, region=ee421baf774947d334ec46882ae42d9c to restore RIT 2024-11-08T00:14:40,378 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910378 2024-11-08T00:14:40,378 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,378 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,380 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,381 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,381 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,382 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,390 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,390 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940390 2024-11-08T00:14:40,390 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000390 2024-11-08T00:14:40,390 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-08T00:14:40,390 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,390 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,390 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,391 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,391 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,391 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,391 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,391 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,392 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,398 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,398 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN 2024-11-08T00:14:40,398 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T00:14:40,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,400 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,400 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,412 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,412 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,412 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,412 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,413 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,413 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,413 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b, ASSIGN to state=OFFLINE, location=null, table=test, region=40e3fed5cf6cc7497a1c2220e1de9b9b to restore RIT 2024-11-08T00:14:40,414 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910414 2024-11-08T00:14:40,414 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,414 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,415 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,416 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,416 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,416 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,422 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,422 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940422 2024-11-08T00:14:40,422 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000422 2024-11-08T00:14:40,422 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T00:14:40,422 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,422 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,423 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,423 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,423 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,423 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,423 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,424 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,426 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,428 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,429 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN 2024-11-08T00:14:40,429 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T00:14:40,430 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,430 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,430 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,445 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,445 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,445 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,445 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,445 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,445 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2ef74f869fdb87a20df612d1e6e2c262, ASSIGN to state=OFFLINE, location=null, table=test, region=2ef74f869fdb87a20df612d1e6e2c262 to restore RIT 2024-11-08T00:14:40,445 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910445 2024-11-08T00:14:40,445 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,445 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,446 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,447 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,448 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,448 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,453 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,454 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,455 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,455 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,455 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940455 2024-11-08T00:14:40,455 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000455 2024-11-08T00:14:40,455 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T00:14:40,455 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,455 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,455 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,455 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,456 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,456 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,456 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,456 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,457 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,459 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,460 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,461 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,461 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,461 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,462 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,462 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN 2024-11-08T00:14:40,462 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T00:14:40,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,463 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,463 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,475 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,475 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,475 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,481 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8, ASSIGN to state=OFFLINE, location=null, table=test, region=faad0b2f1a2327cef2cd0aefa9a4d1c8 to restore RIT 2024-11-08T00:14:40,482 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910482 2024-11-08T00:14:40,482 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,482 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,483 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,484 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,484 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,484 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,489 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,489 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940489 2024-11-08T00:14:40,489 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000489 2024-11-08T00:14:40,489 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-08T00:14:40,489 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,489 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,489 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,489 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,489 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,489 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,489 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,490 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,490 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,492 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,493 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,493 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,494 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,495 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,495 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,495 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,495 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,495 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,495 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,495 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN 2024-11-08T00:14:40,495 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T00:14:40,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,496 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,496 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,509 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a594c54df520141421a62d3f056e15f2, ASSIGN to state=OFFLINE, location=null, table=test, region=a594c54df520141421a62d3f056e15f2 to restore RIT 2024-11-08T00:14:40,510 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910510 2024-11-08T00:14:40,510 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,510 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,511 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,512 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,512 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,512 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,515 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,516 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,517 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,517 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,517 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940517 2024-11-08T00:14:40,517 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000517 2024-11-08T00:14:40,518 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-08T00:14:40,518 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,518 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,518 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,518 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,518 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,518 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,518 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,518 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,518 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,519 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,521 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,522 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,523 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,523 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,523 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,523 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,523 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,523 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,523 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,523 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,523 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,523 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,523 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,523 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN 2024-11-08T00:14:40,523 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-08T00:14:40,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,524 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,524 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,530 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,530 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,530 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,530 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,531 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8280a310195422b4e0184bc7c709a2e9, ASSIGN to state=OFFLINE, location=null, table=test, region=8280a310195422b4e0184bc7c709a2e9 to restore RIT 2024-11-08T00:14:40,531 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910531 2024-11-08T00:14:40,531 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,531 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,532 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,533 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,533 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,533 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,539 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,539 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940539 2024-11-08T00:14:40,539 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000539 2024-11-08T00:14:40,539 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T00:14:40,540 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,540 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,540 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,540 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,540 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,540 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,540 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,540 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,541 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,542 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,543 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,544 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,545 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,545 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,545 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,545 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN 2024-11-08T00:14:40,545 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-08T00:14:40,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,546 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,546 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,585 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,585 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18, ASSIGN to state=OFFLINE, location=null, table=test, region=7acb38c712f4cad61cf5f78f7ffa9a18 to restore RIT 2024-11-08T00:14:40,586 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910586 2024-11-08T00:14:40,586 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,586 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,588 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,589 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,589 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,589 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,595 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,595 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940595 2024-11-08T00:14:40,595 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000595 2024-11-08T00:14:40,595 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T00:14:40,595 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,596 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,596 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,596 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,596 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,596 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,597 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,598 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,601 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN 2024-11-08T00:14:40,602 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T00:14:40,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,603 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,603 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,620 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,624 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,626 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c816073c18c6a09db7883f3602eda012, ASSIGN to state=OFFLINE, location=null, table=test, region=c816073c18c6a09db7883f3602eda012 to restore RIT 2024-11-08T00:14:40,626 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910626 2024-11-08T00:14:40,626 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,626 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,628 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,629 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,629 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,630 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,632 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T00:14:40,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,634 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,634 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940634 2024-11-08T00:14:40,634 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000634 2024-11-08T00:14:40,634 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-08T00:14:40,634 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,634 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,634 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,635 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,635 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,635 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,635 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,635 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,636 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,640 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,640 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T00:14:40,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,642 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,643 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,656 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910656 2024-11-08T00:14:40,656 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,656 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,659 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,660 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,660 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,661 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,665 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,665 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940665 2024-11-08T00:14:40,665 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000665 2024-11-08T00:14:40,665 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T00:14:40,665 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,665 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,665 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,665 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,666 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,666 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,666 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T00:14:40,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,672 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.1731024874653, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,673 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-08T00:14:40,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,673 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,674 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,686 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,687 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910687 2024-11-08T00:14:40,687 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,687 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,692 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,695 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,696 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940695 2024-11-08T00:14:40,696 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000696 2024-11-08T00:14:40,696 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,696 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T00:14:40,696 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,696 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,696 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-08T00:14:40,946 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,946 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,946 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,946 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,947 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,947 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,947 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,948 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T00:14:40,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T00:14:40,950 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,952 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,36139,1731024872925 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,952 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T00:14:40,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,953 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,953 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:40,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:40,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:40,962 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024910962 2024-11-08T00:14:40,962 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:40,962 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,963 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:40,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:40,964 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:40,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:40,969 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:40,969 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024940969 2024-11-08T00:14:40,969 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025000969 2024-11-08T00:14:40,969 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-08T00:14:40,969 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:40,969 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:40,969 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:40,969 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:40,969 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:40,969 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:40,969 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:40,969 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:40,970 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:40,971 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:40,971 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:40,971 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:40,971 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T00:14:40,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:40,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:40,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:40,973 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,973 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:40,973 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:14:40,973 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:40,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:40,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T00:14:40,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:40,973 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:40,974 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:41,005 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,006 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "3302f0f507bd" port: 36139 start_code: 1731024872925 } transition { transition_code: OPENED region_info { region_id: 1731024875468 table_name { namespace: "default" qualifier: "test" } start_key: "ttt" end_key: "uuu" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 74 initiating_master_active_time: 1731024873078 }; retry (#1) after 2000ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T00:14:41,039 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,039 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,055 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024911055 2024-11-08T00:14:41,055 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:41,055 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,058 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:41,059 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:41,059 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:41,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:41,059 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,065 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:41,065 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024941065 2024-11-08T00:14:41,065 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025001065 2024-11-08T00:14:41,065 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-08T00:14:41,065 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:41,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:41,065 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:41,065 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:41,065 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:41,065 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:41,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:41,066 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:41,066 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:41,068 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:41,068 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:41,068 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T00:14:41,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:41,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:41,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:41,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:41,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:41,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:41,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:41,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:41,069 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-08T00:14:41,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:41,070 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:41,070 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:41,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,078 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024911078 2024-11-08T00:14:41,078 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:41,078 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,079 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:41,080 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:41,080 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:41,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:41,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,085 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:41,086 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024941085 2024-11-08T00:14:41,086 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025001086 2024-11-08T00:14:41,086 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-08T00:14:41,086 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:41,086 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:41,086 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:41,086 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:41,086 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:41,086 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:41,086 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:41,087 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:41,087 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:41,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:41,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T00:14:41,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:41,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:41,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:41,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 2024-11-08T00:14:41,091 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:41,091 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3302f0f507bd%2C44033%2C1731024872883.meta.1731024875081.meta, worker=3302f0f507bd,38371,1731024872767 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T00:14:41,091 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-08T00:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:41,092 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:41,092 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:41,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,099 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024911099 2024-11-08T00:14:41,099 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:41,099 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,100 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:41,101 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-08T00:14:41,101 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:41,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:41,101 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,105 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:41,105 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024941105 2024-11-08T00:14:41,105 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025001105 2024-11-08T00:14:41,105 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-08T00:14:41,105 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T00:14:41,105 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:41,105 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:41,105 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:41,105 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T00:14:41,105 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T00:14:41,105 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T00:14:41,105 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:14:41,106 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T00:14:41,107 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T00:14:41,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:14:41,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-08T00:14:41,108 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true 2024-11-08T00:14:41,108 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T00:14:41,108 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 1 msec 2024-11-08T00:14:41,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=61 2024-11-08T00:14:41,109 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3302f0f507bd,36139,1731024872925, table=hbase:meta, region=1588230740 2024-11-08T00:14:41,109 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,36139,1731024872925, state=OPEN 2024-11-08T00:14:41,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:14:41,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:14:41,120 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731024911120 2024-11-08T00:14:41,120 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:14:41,120 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:14:41,122 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-08T00:14:41,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure a594c54df520141421a62d3f056e15f2, server=3302f0f507bd,38371,1731024872767:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 3302f0f507bd,44033,1731024872883, splitWal=true, meta=true exec-time=2.9690 sec 2024-11-08T00:14:41,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8624a8443e655fa753c4de0934012d54, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a594c54df520141421a62d3f056e15f2, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9c5dafc4711af09b862954533a7784ca, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ffaf29acbc927f398077e41ad232c549, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cd2627434f69d79290b6a35838b5f326, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66bf32aac498176881371b724c9d8008, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=faad0b2f1a2327cef2cd0aefa9a4d1c8, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5026dfa3bad2942533f117d619b1656, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13f20a01a04306fecc9b545706c9d14c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ef74f869fdb87a20df612d1e6e2c262, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f04142b3a962ffb3cc6e9629a0aa3c6a, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bb82385d0b72eb69ca4bdc52663bf8a, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c8336e737af786ef3c90e95f98c755f, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee421baf774947d334ec46882ae42d9c, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3bddcc13c4eccc1d32b3c09b1dcb1583, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=850fbeeee9e2784d3676499ac7582114, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7acb38c712f4cad61cf5f78f7ffa9a18, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8e8ac9592d4e973f9896a6807e10272, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e10c5ec7fdc210db94ac8df9187f1c4, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8280a310195422b4e0184bc7c709a2e9, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54a0ce095f1e6524de8bdf904569509f, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcb90ad7cb983c915be269c865806709, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=40e3fed5cf6cc7497a1c2220e1de9b9b, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=335e583f3dd0eda08c4c42b80a743311, regionState=OPEN, lastHost=3302f0f507bd,36139,1731024872925, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f09fb3c8c48c1b2f01441065e691680c, regionState=OPEN, lastHost=3302f0f507bd,38371,1731024872767, regionLocation=3302f0f507bd,38371,1731024872767, openSeqNum=2 2024-11-08T00:14:41,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c816073c18c6a09db7883f3602eda012, regionState=OPENING, lastHost=3302f0f507bd,44033,1731024872883, regionLocation=3302f0f507bd,36139,1731024872925, openSeqNum=2 2024-11-08T00:14:41,129 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-08T00:14:41,129 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731024941129 2024-11-08T00:14:41,129 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025001129 2024-11-08T00:14:41,129 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-08T00:14:41,147 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=345 (was 373), OpenFileDescriptor=586 (was 605), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=908 (was 926), ProcessCount=11 (was 11), AvailableMemoryMB=1089 (was 1176) 2024-11-08T00:14:41,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T00:14:41,147 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:14:41,148 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:14:41,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:41,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:41,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:14:41,149 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T00:14:41,149 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=840105357, stopped=false 2024-11-08T00:14:41,149 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3302f0f507bd,36429,1731024871043 2024-11-08T00:14:41,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:14:41,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:14:41,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:14:41,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:41,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:41,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:41,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:41,162 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:14:41,162 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:14:41,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:41,162 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3302f0f507bd,38371,1731024872767' ***** 2024-11-08T00:14:41,162 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T00:14:41,162 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:14:41,162 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:14:41,162 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3302f0f507bd,36139,1731024872925' ***** 2024-11-08T00:14:41,162 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T00:14:41,162 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T00:14:41,162 INFO [RS:0;3302f0f507bd:38371 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T00:14:41,162 INFO [RS:0;3302f0f507bd:38371 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T00:14:41,162 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T00:14:41,162 INFO [RS:2;3302f0f507bd:36139 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T00:14:41,162 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T00:14:41,163 INFO [RS:2;3302f0f507bd:36139 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T00:14:41,163 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for a594c54df520141421a62d3f056e15f2 2024-11-08T00:14:41,163 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:14:41,163 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for cd2627434f69d79290b6a35838b5f326 2024-11-08T00:14:41,163 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T00:14:41,163 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 2ef74f869fdb87a20df612d1e6e2c262 2024-11-08T00:14:41,163 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 40e3fed5cf6cc7497a1c2220e1de9b9b 2024-11-08T00:14:41,163 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for b8e8ac9592d4e973f9896a6807e10272 2024-11-08T00:14:41,163 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 66bf32aac498176881371b724c9d8008 2024-11-08T00:14:41,163 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for 8624a8443e655fa753c4de0934012d54 2024-11-08T00:14:41,163 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 9c5dafc4711af09b862954533a7784ca 2024-11-08T00:14:41,163 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:41,163 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 13f20a01a04306fecc9b545706c9d14c 2024-11-08T00:14:41,163 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for f04142b3a962ffb3cc6e9629a0aa3c6a 2024-11-08T00:14:41,163 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 3bb82385d0b72eb69ca4bdc52663bf8a 2024-11-08T00:14:41,163 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for 3bddcc13c4eccc1d32b3c09b1dcb1583 2024-11-08T00:14:41,163 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:41,163 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for b5026dfa3bad2942533f117d619b1656 2024-11-08T00:14:41,163 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for bcb90ad7cb983c915be269c865806709 2024-11-08T00:14:41,164 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 850fbeeee9e2784d3676499ac7582114 2024-11-08T00:14:41,164 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for 335e583f3dd0eda08c4c42b80a743311 2024-11-08T00:14:41,164 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 2e10c5ec7fdc210db94ac8df9187f1c4 2024-11-08T00:14:41,164 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for 54a0ce095f1e6524de8bdf904569509f 2024-11-08T00:14:41,164 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 7acb38c712f4cad61cf5f78f7ffa9a18 2024-11-08T00:14:41,164 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(3091): Received CLOSE for 1c8336e737af786ef3c90e95f98c755f 2024-11-08T00:14:41,164 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a594c54df520141421a62d3f056e15f2, disabling compactions & flushes 2024-11-08T00:14:41,164 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cd2627434f69d79290b6a35838b5f326, disabling compactions & flushes 2024-11-08T00:14:41,164 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(959): stopping server 3302f0f507bd,36139,1731024872925 2024-11-08T00:14:41,164 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(959): stopping server 3302f0f507bd,38371,1731024872767 2024-11-08T00:14:41,164 INFO [RS:0;3302f0f507bd:38371 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:14:41,164 INFO [RS:2;3302f0f507bd:36139 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:14:41,164 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:41,164 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:41,164 INFO [RS:0;3302f0f507bd:38371 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3302f0f507bd:38371. 2024-11-08T00:14:41,164 INFO [RS:2;3302f0f507bd:36139 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;3302f0f507bd:36139. 2024-11-08T00:14:41,164 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:41,164 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:41,164 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. after waiting 0 ms 2024-11-08T00:14:41,164 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. after waiting 0 ms 2024-11-08T00:14:41,164 DEBUG [RS:2;3302f0f507bd:36139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:14:41,164 DEBUG [RS:0;3302f0f507bd:38371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:14:41,164 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:41,164 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:41,164 DEBUG [RS:2;3302f0f507bd:36139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:41,164 DEBUG [RS:0;3302f0f507bd:38371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:41,164 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-08T00:14:41,164 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T00:14:41,164 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T00:14:41,164 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T00:14:41,164 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T00:14:41,165 DEBUG [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(1351): Waiting on 13f20a01a04306fecc9b545706c9d14c, 1c8336e737af786ef3c90e95f98c755f, 2e10c5ec7fdc210db94ac8df9187f1c4, 2ef74f869fdb87a20df612d1e6e2c262, 3bb82385d0b72eb69ca4bdc52663bf8a, 40e3fed5cf6cc7497a1c2220e1de9b9b, 66bf32aac498176881371b724c9d8008, 7acb38c712f4cad61cf5f78f7ffa9a18, 850fbeeee9e2784d3676499ac7582114, 9c5dafc4711af09b862954533a7784ca, a594c54df520141421a62d3f056e15f2, bcb90ad7cb983c915be269c865806709, f09fb3c8c48c1b2f01441065e691680c 2024-11-08T00:14:41,165 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-08T00:14:41,165 DEBUG [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 335e583f3dd0eda08c4c42b80a743311, 3bddcc13c4eccc1d32b3c09b1dcb1583, 54a0ce095f1e6524de8bdf904569509f, 8280a310195422b4e0184bc7c709a2e9, 8624a8443e655fa753c4de0934012d54, b5026dfa3bad2942533f117d619b1656, b8e8ac9592d4e973f9896a6807e10272, cd2627434f69d79290b6a35838b5f326, f04142b3a962ffb3cc6e9629a0aa3c6a, ffaf29acbc927f398077e41ad232c549 2024-11-08T00:14:41,165 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T00:14:41,165 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T00:14:41,165 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T00:14:41,165 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T00:14:41,165 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T00:14:41,165 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-11-08T00:14:41,169 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/cd2627434f69d79290b6a35838b5f326/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,169 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/a594c54df520141421a62d3f056e15f2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,171 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:41,171 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:41,171 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cd2627434f69d79290b6a35838b5f326: Waiting for close lock at 1731024881163Running coprocessor pre-close hooks at 1731024881164 (+1 ms)Disabling compacts and flushes for region at 1731024881164Disabling writes for close at 1731024881164Writing region close event to WAL at 1731024881165 (+1 ms)Running coprocessor post-close hooks at 1731024881169 (+4 ms)Closed at 1731024881171 (+2 ms) 2024-11-08T00:14:41,171 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a594c54df520141421a62d3f056e15f2: Waiting for close lock at 1731024881163Running coprocessor pre-close hooks at 1731024881164 (+1 ms)Disabling compacts and flushes for region at 1731024881164Disabling writes for close at 1731024881164Writing region close event to WAL at 1731024881165 (+1 ms)Running coprocessor post-close hooks at 1731024881169 (+4 ms)Closed at 1731024881171 (+2 ms) 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731024875468.cd2627434f69d79290b6a35838b5f326. 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2. 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b8e8ac9592d4e973f9896a6807e10272, disabling compactions & flushes 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2ef74f869fdb87a20df612d1e6e2c262, disabling compactions & flushes 2024-11-08T00:14:41,172 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:41,172 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. after waiting 0 ms 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. after waiting 0 ms 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:41,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:41,177 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/b8e8ac9592d4e973f9896a6807e10272/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,178 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:41,178 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b8e8ac9592d4e973f9896a6807e10272: Waiting for close lock at 1731024881172Running coprocessor pre-close hooks at 1731024881172Disabling compacts and flushes for region at 1731024881172Disabling writes for close at 1731024881172Writing region close event to WAL at 1731024881172Running coprocessor post-close hooks at 1731024881178 (+6 ms)Closed at 1731024881178 2024-11-08T00:14:41,178 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731024875468.b8e8ac9592d4e973f9896a6807e10272. 2024-11-08T00:14:41,178 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8624a8443e655fa753c4de0934012d54, disabling compactions & flushes 2024-11-08T00:14:41,178 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:41,178 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:41,179 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731024875468.8624a8443e655fa753c4de0934012d54. after waiting 0 ms 2024-11-08T00:14:41,179 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:41,180 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/info/c0c1797bee7b4406bf48dccfb541b259 is 138, key is test,bbb,1731024875468.a594c54df520141421a62d3f056e15f2./info:regioninfo/1731024879670/Put/seqid=0 2024-11-08T00:14:41,184 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/8624a8443e655fa753c4de0934012d54/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,184 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2ef74f869fdb87a20df612d1e6e2c262/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,185 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:41,185 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8624a8443e655fa753c4de0934012d54: Waiting for close lock at 1731024881178Running coprocessor pre-close hooks at 1731024881178Disabling compacts and flushes for region at 1731024881178Disabling writes for close at 1731024881179 (+1 ms)Writing region close event to WAL at 1731024881180 (+1 ms)Running coprocessor post-close hooks at 1731024881185 (+5 ms)Closed at 1731024881185 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2ef74f869fdb87a20df612d1e6e2c262: Waiting for close lock at 1731024881172Running coprocessor pre-close hooks at 1731024881172Disabling compacts and flushes for region at 1731024881172Disabling writes for close at 1731024881172Writing region close event to WAL at 1731024881175 (+3 ms)Running coprocessor post-close hooks at 1731024881185 (+10 ms)Closed at 1731024881185 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731024875468.2ef74f869fdb87a20df612d1e6e2c262. 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731024875468.8624a8443e655fa753c4de0934012d54. 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40e3fed5cf6cc7497a1c2220e1de9b9b, disabling compactions & flushes 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ffaf29acbc927f398077e41ad232c549, disabling compactions & flushes 2024-11-08T00:14:41,185 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:41,185 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. after waiting 0 ms 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. after waiting 0 ms 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:41,185 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:41,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741869_1045 (size=8192) 2024-11-08T00:14:41,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741869_1045 (size=8192) 2024-11-08T00:14:41,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741869_1045 (size=8192) 2024-11-08T00:14:41,194 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/ffaf29acbc927f398077e41ad232c549/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,195 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/info/c0c1797bee7b4406bf48dccfb541b259 2024-11-08T00:14:41,195 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:41,195 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/40e3fed5cf6cc7497a1c2220e1de9b9b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,195 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ffaf29acbc927f398077e41ad232c549: Waiting for close lock at 1731024881185Running coprocessor pre-close hooks at 1731024881185Disabling compacts and flushes for region at 1731024881185Disabling writes for close at 1731024881185Writing region close event to WAL at 1731024881188 (+3 ms)Running coprocessor post-close hooks at 1731024881195 (+7 ms)Closed at 1731024881195 2024-11-08T00:14:41,195 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731024875468.ffaf29acbc927f398077e41ad232c549. 2024-11-08T00:14:41,195 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f04142b3a962ffb3cc6e9629a0aa3c6a, disabling compactions & flushes 2024-11-08T00:14:41,195 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:41,195 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:41,196 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. after waiting 0 ms 2024-11-08T00:14:41,196 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:41,196 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:41,196 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40e3fed5cf6cc7497a1c2220e1de9b9b: Waiting for close lock at 1731024881185Running coprocessor pre-close hooks at 1731024881185Disabling compacts and flushes for region at 1731024881185Disabling writes for close at 1731024881185Writing region close event to WAL at 1731024881188 (+3 ms)Running coprocessor post-close hooks at 1731024881196 (+8 ms)Closed at 1731024881196 2024-11-08T00:14:41,196 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731024875468.40e3fed5cf6cc7497a1c2220e1de9b9b. 2024-11-08T00:14:41,197 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 66bf32aac498176881371b724c9d8008, disabling compactions & flushes 2024-11-08T00:14:41,197 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:41,197 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:41,197 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731024875468.66bf32aac498176881371b724c9d8008. after waiting 0 ms 2024-11-08T00:14:41,197 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:41,201 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/f04142b3a962ffb3cc6e9629a0aa3c6a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,202 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:41,202 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f04142b3a962ffb3cc6e9629a0aa3c6a: Waiting for close lock at 1731024881195Running coprocessor pre-close hooks at 1731024881195Disabling compacts and flushes for region at 1731024881195Disabling writes for close at 1731024881196 (+1 ms)Writing region close event to WAL at 1731024881196Running coprocessor post-close hooks at 1731024881202 (+6 ms)Closed at 1731024881202 2024-11-08T00:14:41,202 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/66bf32aac498176881371b724c9d8008/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,202 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731024875468.f04142b3a962ffb3cc6e9629a0aa3c6a. 2024-11-08T00:14:41,202 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3bddcc13c4eccc1d32b3c09b1dcb1583, disabling compactions & flushes 2024-11-08T00:14:41,202 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:41,202 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:41,203 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. after waiting 0 ms 2024-11-08T00:14:41,203 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:41,203 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:41,203 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 66bf32aac498176881371b724c9d8008: Waiting for close lock at 1731024881197Running coprocessor pre-close hooks at 1731024881197Disabling compacts and flushes for region at 1731024881197Disabling writes for close at 1731024881197Writing region close event to WAL at 1731024881197Running coprocessor post-close hooks at 1731024881203 (+6 ms)Closed at 1731024881203 2024-11-08T00:14:41,204 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731024875468.66bf32aac498176881371b724c9d8008. 2024-11-08T00:14:41,204 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9c5dafc4711af09b862954533a7784ca, disabling compactions & flushes 2024-11-08T00:14:41,204 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:41,204 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:41,204 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. after waiting 0 ms 2024-11-08T00:14:41,204 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:41,209 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/9c5dafc4711af09b862954533a7784ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,209 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/.tmp/info/c0c1797bee7b4406bf48dccfb541b259 as hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/info/c0c1797bee7b4406bf48dccfb541b259 2024-11-08T00:14:41,209 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/3bddcc13c4eccc1d32b3c09b1dcb1583/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,210 INFO [regionserver/3302f0f507bd:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:14:41,210 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:41,210 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9c5dafc4711af09b862954533a7784ca: Waiting for close lock at 1731024881204Running coprocessor pre-close hooks at 1731024881204Disabling compacts and flushes for region at 1731024881204Disabling writes for close at 1731024881204Writing region close event to WAL at 1731024881205 (+1 ms)Running coprocessor post-close hooks at 1731024881210 (+5 ms)Closed at 1731024881210 2024-11-08T00:14:41,210 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731024875468.9c5dafc4711af09b862954533a7784ca. 2024-11-08T00:14:41,210 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:41,210 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3bddcc13c4eccc1d32b3c09b1dcb1583: Waiting for close lock at 1731024881202Running coprocessor pre-close hooks at 1731024881202Disabling compacts and flushes for region at 1731024881202Disabling writes for close at 1731024881203 (+1 ms)Writing region close event to WAL at 1731024881203Running coprocessor post-close hooks at 1731024881210 (+7 ms)Closed at 1731024881210 2024-11-08T00:14:41,210 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 13f20a01a04306fecc9b545706c9d14c, disabling compactions & flushes 2024-11-08T00:14:41,211 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:41,211 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:41,211 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731024875468.3bddcc13c4eccc1d32b3c09b1dcb1583. 2024-11-08T00:14:41,211 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. after waiting 0 ms 2024-11-08T00:14:41,211 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:41,211 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b5026dfa3bad2942533f117d619b1656, disabling compactions & flushes 2024-11-08T00:14:41,211 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:41,211 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:41,211 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. after waiting 0 ms 2024-11-08T00:14:41,211 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:41,211 INFO [regionserver/3302f0f507bd:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:14:41,215 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/b5026dfa3bad2942533f117d619b1656/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,216 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:41,216 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b5026dfa3bad2942533f117d619b1656: Waiting for close lock at 1731024881211Running coprocessor pre-close hooks at 1731024881211Disabling compacts and flushes for region at 1731024881211Disabling writes for close at 1731024881211Writing region close event to WAL at 1731024881211Running coprocessor post-close hooks at 1731024881216 (+5 ms)Closed at 1731024881216 2024-11-08T00:14:41,216 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731024875468.b5026dfa3bad2942533f117d619b1656. 2024-11-08T00:14:41,216 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 335e583f3dd0eda08c4c42b80a743311, disabling compactions & flushes 2024-11-08T00:14:41,217 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:41,217 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:41,217 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. after waiting 0 ms 2024-11-08T00:14:41,217 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:41,218 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/13f20a01a04306fecc9b545706c9d14c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,219 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:41,219 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/info/c0c1797bee7b4406bf48dccfb541b259, entries=24, sequenceid=71, filesize=8 K 2024-11-08T00:14:41,219 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 13f20a01a04306fecc9b545706c9d14c: Waiting for close lock at 1731024881210Running coprocessor pre-close hooks at 1731024881210Disabling compacts and flushes for region at 1731024881210Disabling writes for close at 1731024881211 (+1 ms)Writing region close event to WAL at 1731024881212 (+1 ms)Running coprocessor post-close hooks at 1731024881219 (+7 ms)Closed at 1731024881219 2024-11-08T00:14:41,219 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731024875468.13f20a01a04306fecc9b545706c9d14c. 2024-11-08T00:14:41,219 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3bb82385d0b72eb69ca4bdc52663bf8a, disabling compactions & flushes 2024-11-08T00:14:41,219 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:41,219 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:41,219 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. after waiting 0 ms 2024-11-08T00:14:41,220 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:41,220 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 55ms, sequenceid=71, compaction requested=false 2024-11-08T00:14:41,222 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/335e583f3dd0eda08c4c42b80a743311/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,223 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:41,224 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 335e583f3dd0eda08c4c42b80a743311: Waiting for close lock at 1731024881216Running coprocessor pre-close hooks at 1731024881216Disabling compacts and flushes for region at 1731024881216Disabling writes for close at 1731024881217 (+1 ms)Writing region close event to WAL at 1731024881218 (+1 ms)Running coprocessor post-close hooks at 1731024881223 (+5 ms)Closed at 1731024881223 2024-11-08T00:14:41,224 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731024875468.335e583f3dd0eda08c4c42b80a743311. 2024-11-08T00:14:41,224 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 54a0ce095f1e6524de8bdf904569509f, disabling compactions & flushes 2024-11-08T00:14:41,224 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:41,224 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:41,224 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. after waiting 0 ms 2024-11-08T00:14:41,224 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:41,229 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/3bb82385d0b72eb69ca4bdc52663bf8a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,230 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:41,230 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3bb82385d0b72eb69ca4bdc52663bf8a: Waiting for close lock at 1731024881219Running coprocessor pre-close hooks at 1731024881219Disabling compacts and flushes for region at 1731024881219Disabling writes for close at 1731024881219Writing region close event to WAL at 1731024881220 (+1 ms)Running coprocessor post-close hooks at 1731024881230 (+10 ms)Closed at 1731024881230 2024-11-08T00:14:41,230 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731024875468.3bb82385d0b72eb69ca4bdc52663bf8a. 2024-11-08T00:14:41,230 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f09fb3c8c48c1b2f01441065e691680c, disabling compactions & flushes 2024-11-08T00:14:41,230 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:41,230 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:41,230 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. after waiting 0 ms 2024-11-08T00:14:41,230 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:41,232 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-08T00:14:41,232 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/54a0ce095f1e6524de8bdf904569509f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,233 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T00:14:41,233 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:41,233 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T00:14:41,233 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 54a0ce095f1e6524de8bdf904569509f: Waiting for close lock at 1731024881224Running coprocessor pre-close hooks at 1731024881224Disabling compacts and flushes for region at 1731024881224Disabling writes for close at 1731024881224Writing region close event to WAL at 1731024881228 (+4 ms)Running coprocessor post-close hooks at 1731024881233 (+5 ms)Closed at 1731024881233 2024-11-08T00:14:41,233 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731024881165Running coprocessor pre-close hooks at 1731024881165Disabling compacts and flushes for region at 1731024881165Disabling writes for close at 1731024881165Obtaining lock to block concurrent updates at 1731024881165Preparing flush snapshotting stores in 1588230740 at 1731024881165Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731024881173 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731024881175 (+2 ms)Flushing 1588230740/info: creating writer at 1731024881175Flushing 1588230740/info: appending metadata at 1731024881179 (+4 ms)Flushing 1588230740/info: closing flushed file at 1731024881179Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27d8bc9a: reopening flushed file at 1731024881208 (+29 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 55ms, sequenceid=71, compaction requested=false at 1731024881220 (+12 ms)Writing region close event to WAL at 1731024881224 (+4 ms)Running coprocessor post-close hooks at 1731024881233 (+9 ms)Closed at 1731024881233 2024-11-08T00:14:41,233 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731024875468.54a0ce095f1e6524de8bdf904569509f. 2024-11-08T00:14:41,233 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T00:14:41,235 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/f09fb3c8c48c1b2f01441065e691680c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,235 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:41,236 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f09fb3c8c48c1b2f01441065e691680c: Waiting for close lock at 1731024881230Running coprocessor pre-close hooks at 1731024881230Disabling compacts and flushes for region at 1731024881230Disabling writes for close at 1731024881230Writing region close event to WAL at 1731024881231 (+1 ms)Running coprocessor post-close hooks at 1731024881235 (+4 ms)Closed at 1731024881235 2024-11-08T00:14:41,236 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731024875468.f09fb3c8c48c1b2f01441065e691680c. 2024-11-08T00:14:41,236 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bcb90ad7cb983c915be269c865806709, disabling compactions & flushes 2024-11-08T00:14:41,236 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:41,236 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:41,236 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. after waiting 0 ms 2024-11-08T00:14:41,236 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:41,240 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/bcb90ad7cb983c915be269c865806709/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,241 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:41,241 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bcb90ad7cb983c915be269c865806709: Waiting for close lock at 1731024881236Running coprocessor pre-close hooks at 1731024881236Disabling compacts and flushes for region at 1731024881236Disabling writes for close at 1731024881236Writing region close event to WAL at 1731024881236Running coprocessor post-close hooks at 1731024881240 (+4 ms)Closed at 1731024881241 (+1 ms) 2024-11-08T00:14:41,241 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731024875468.bcb90ad7cb983c915be269c865806709. 2024-11-08T00:14:41,241 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 850fbeeee9e2784d3676499ac7582114, disabling compactions & flushes 2024-11-08T00:14:41,241 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:41,241 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:41,241 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. after waiting 0 ms 2024-11-08T00:14:41,241 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:41,245 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/850fbeeee9e2784d3676499ac7582114/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,246 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:41,246 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 850fbeeee9e2784d3676499ac7582114: Waiting for close lock at 1731024881241Running coprocessor pre-close hooks at 1731024881241Disabling compacts and flushes for region at 1731024881241Disabling writes for close at 1731024881241Writing region close event to WAL at 1731024881241Running coprocessor post-close hooks at 1731024881246 (+5 ms)Closed at 1731024881246 2024-11-08T00:14:41,246 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731024875468.850fbeeee9e2784d3676499ac7582114. 2024-11-08T00:14:41,246 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2e10c5ec7fdc210db94ac8df9187f1c4, disabling compactions & flushes 2024-11-08T00:14:41,246 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:41,246 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:41,246 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. after waiting 0 ms 2024-11-08T00:14:41,247 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:41,253 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/2e10c5ec7fdc210db94ac8df9187f1c4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,254 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:41,254 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2e10c5ec7fdc210db94ac8df9187f1c4: Waiting for close lock at 1731024881246Running coprocessor pre-close hooks at 1731024881246Disabling compacts and flushes for region at 1731024881246Disabling writes for close at 1731024881246Writing region close event to WAL at 1731024881248 (+2 ms)Running coprocessor post-close hooks at 1731024881254 (+6 ms)Closed at 1731024881254 2024-11-08T00:14:41,255 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731024875468.2e10c5ec7fdc210db94ac8df9187f1c4. 2024-11-08T00:14:41,255 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7acb38c712f4cad61cf5f78f7ffa9a18, disabling compactions & flushes 2024-11-08T00:14:41,255 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:41,255 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:41,255 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. after waiting 0 ms 2024-11-08T00:14:41,255 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:41,260 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/7acb38c712f4cad61cf5f78f7ffa9a18/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,260 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:41,261 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7acb38c712f4cad61cf5f78f7ffa9a18: Waiting for close lock at 1731024881255Running coprocessor pre-close hooks at 1731024881255Disabling compacts and flushes for region at 1731024881255Disabling writes for close at 1731024881255Writing region close event to WAL at 1731024881256 (+1 ms)Running coprocessor post-close hooks at 1731024881260 (+4 ms)Closed at 1731024881260 2024-11-08T00:14:41,261 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731024875468.7acb38c712f4cad61cf5f78f7ffa9a18. 2024-11-08T00:14:41,261 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1c8336e737af786ef3c90e95f98c755f, disabling compactions & flushes 2024-11-08T00:14:41,261 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:41,261 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:41,261 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. after waiting 0 ms 2024-11-08T00:14:41,261 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:41,266 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/data/default/test/1c8336e737af786ef3c90e95f98c755f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:14:41,267 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:41,267 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1c8336e737af786ef3c90e95f98c755f: Waiting for close lock at 1731024881261Running coprocessor pre-close hooks at 1731024881261Disabling compacts and flushes for region at 1731024881261Disabling writes for close at 1731024881261Writing region close event to WAL at 1731024881261Running coprocessor post-close hooks at 1731024881266 (+5 ms)Closed at 1731024881266 2024-11-08T00:14:41,267 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731024875468.1c8336e737af786ef3c90e95f98c755f. 2024-11-08T00:14:41,365 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(976): stopping server 3302f0f507bd,36139,1731024872925; all regions closed. 2024-11-08T00:14:41,365 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(976): stopping server 3302f0f507bd,38371,1731024872767; all regions closed. 2024-11-08T00:14:41,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741834_1010 (size=7074) 2024-11-08T00:14:41,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741834_1010 (size=7074) 2024-11-08T00:14:41,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741865_1041 (size=4201) 2024-11-08T00:14:41,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741865_1041 (size=4201) 2024-11-08T00:14:41,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741865_1041 (size=4201) 2024-11-08T00:14:41,376 DEBUG [RS:0;3302f0f507bd:38371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs 2024-11-08T00:14:41,377 INFO [RS:0;3302f0f507bd:38371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3302f0f507bd%2C38371%2C1731024872767:(num 1731024874653) 2024-11-08T00:14:41,377 DEBUG [RS:2;3302f0f507bd:36139 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs 2024-11-08T00:14:41,377 DEBUG [RS:0;3302f0f507bd:38371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:41,377 INFO [RS:2;3302f0f507bd:36139 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3302f0f507bd%2C36139%2C1731024872925.meta:.meta(num 1731024878833) 2024-11-08T00:14:41,377 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:14:41,377 INFO [RS:0;3302f0f507bd:38371 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:14:41,377 INFO [RS:0;3302f0f507bd:38371 {}] hbase.ChoreService(370): Chore service for: regionserver/3302f0f507bd:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T00:14:41,377 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T00:14:41,377 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T00:14:41,377 INFO [regionserver/3302f0f507bd:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:14:41,377 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T00:14:41,377 INFO [RS:0;3302f0f507bd:38371 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:14:41,378 INFO [RS:0;3302f0f507bd:38371 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38371 2024-11-08T00:14:41,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741835_1011 (size=5184) 2024-11-08T00:14:41,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741835_1011 (size=5184) 2024-11-08T00:14:41,382 DEBUG [RS:2;3302f0f507bd:36139 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/oldWALs 2024-11-08T00:14:41,382 INFO [RS:2;3302f0f507bd:36139 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3302f0f507bd%2C36139%2C1731024872925:(num 1731024874653) 2024-11-08T00:14:41,382 DEBUG [RS:2;3302f0f507bd:36139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:14:41,382 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:14:41,382 INFO [RS:2;3302f0f507bd:36139 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:14:41,382 INFO [RS:2;3302f0f507bd:36139 {}] hbase.ChoreService(370): Chore service for: regionserver/3302f0f507bd:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T00:14:41,382 INFO [RS:2;3302f0f507bd:36139 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:14:41,382 INFO [regionserver/3302f0f507bd:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:14:41,382 WARN [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:14:41,383 WARN [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9., aborting... java.io.IOException: Failed to report opened region to master: test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:14:41,383 ERROR [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2440): ***** ABORTING region server 3302f0f507bd,36139,1731024872925: Failed to open region test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ttt,1731024875468.8280a310195422b4e0184bc7c709a2e9. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:14:41,383 ERROR [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-08T00:14:41,383 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-08T00:14:41,384 INFO [RS:2;3302f0f507bd:36139 {}] executor.ExecutorService(108): Executor-10-RS_OPEN_REGION-regionserver/3302f0f507bd:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-08T00:14:41,384 INFO [RS:2;3302f0f507bd:36139 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36139 2024-11-08T00:14:41,384 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-08T00:14:41,386 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-08T00:14:41,387 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-08T00:14:41,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3302f0f507bd,38371,1731024872767 2024-11-08T00:14:41,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:14:41,391 INFO [RS:0;3302f0f507bd:38371 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:14:41,395 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 475137776 }, "NonHeapMemoryUsage": { "committed": 163250176, "init": 7667712, "max": -1, "used": 160575528 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "3302f0f507bd", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 40173, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 128, "ProcessCallTime_min": 0, "ProcessCallTime_max": 38, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 9, "ProcessCallTime_median": 19, "ProcessCallTime_75th_percentile": 28, "ProcessCallTime_90th_percentile": 34, "ProcessCallTime_95th_percentile": 36, "ProcessCallTime_98th_percentile": 37, "ProcessCallTime_99th_percentile": 37, "ProcessCallTime_99.9th_percentile": 37, "ProcessCallTime_TimeRangeCount_0-1": 128, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 128, "QueueCallTime_min": 0, "QueueCallTime_max": 14, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 7, "QueueCallTime_75th_percentile": 10, "QueueCallTime_90th_percentile": 12, "QueueCallTime_95th_percentile": 13, "QueueCallTime_98th_percentile": 13, "QueueCallTime_99th_percentile": 13, "QueueCallTime_99.9th_percentile": 13, "QueueCallTime_TimeRangeCount_0-1": 128, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 128, "TotalCallTime_min": 0, "TotalCallTime_max": 38, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 9, "TotalCallTime_median": 19, "TotalCallTime_75th_percentile": 28, "TotalCallTime_90th_percentile": 34, "TotalCallTime_95th_percentile": 36, "TotalCallTime_98th_percentile": 37, "TotalCallTime_99th_percentile": 37, "TotalCallTime_99.9th_percentile": 37, "TotalCallTime_TimeRangeCount_0-1": 128, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 128, "ResponseSize_min": 0, "ResponseSize_max": 22858, "ResponseSize_mean": 3755, "ResponseSize_25th_percentile": 5714, "ResponseSize_median": 11429, "ResponseSize_75th_percentile": 17143, "ResponseSize_90th_percentile": 20572, "ResponseSize_95th_percentile": 21715, "ResponseSize_98th_percentile": 22400, "ResponseSize_99th_percentile": 22629, "ResponseSize_99.9th_percentile": 22835, "ResponseSize_SizeRangeCount_0-10": 128, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 128, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 284, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 128, "sentBytes": 372562 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "3302f0f507bd", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:54358", "tag.serverName": "3302f0f507bd,44033,1731024872883", "tag.clusterId": "2dc56221-06a4-403e-ac1f-b12cec23f67d", "tag.Context": "regionserver", "tag.Hostname": "3302f0f507bd", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 28672, "memStoreHeapSize": 51520, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731024872883, "averageRegionSize": 3185, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 19.0, "writeRequestRatePerSecond": 16.400000000000002, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 532480, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 75, "totalRowActionRequestCount": 177, "readRequestCount": 95, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 82, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 56, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 9, "ScanTime_mean": 1, "ScanTime_25th_percentile": 2, "ScanTime_median": 4, "ScanTime_75th_percentile": 6, "ScanTime_90th_percentile": 8, "ScanTime_95th_percentile": 8, "ScanTime_98th_percentile": 8, "ScanTime_99th_percentile": 8, "ScanTime_99.9th_percentile": 8, "ScanTime_TimeRangeCount_0-1": 16, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 56, "Put_min": 1, "Put_max": 7, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 56, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 9, "PutBatch_max": 9, "PutBatch_mean": 9, "PutBatch_25th_percentile": 9, "PutBatch_median": 9, "PutBatch_75th_percentile": 9, "PutBatch_90th_percentile": 9, "PutBatch_95th_percentile": 9, "PutBatch_98th_percentile": 9, "PutBatch_99th_percentile": 9, "PutBatch_99.9th_percentile": 9, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 1, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4238, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 16, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-08T00:14:41,401 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3302f0f507bd,38371,1731024872767] 2024-11-08T00:14:41,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3302f0f507bd,36139,1731024872925 2024-11-08T00:14:41,401 INFO [RS:2;3302f0f507bd:36139 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:14:41,422 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3302f0f507bd,38371,1731024872767 already deleted, retry=false 2024-11-08T00:14:41,422 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3302f0f507bd,38371,1731024872767 expired; onlineServers=1 2024-11-08T00:14:41,422 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3302f0f507bd,36139,1731024872925] 2024-11-08T00:14:41,435 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3302f0f507bd,36139,1731024872925 already deleted, retry=false 2024-11-08T00:14:41,435 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3302f0f507bd,36139,1731024872925 expired; onlineServers=0 2024-11-08T00:14:41,435 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3302f0f507bd,36429,1731024871043' ***** 2024-11-08T00:14:41,435 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T00:14:41,435 INFO [M:0;3302f0f507bd:36429 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:14:41,435 INFO [M:0;3302f0f507bd:36429 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:14:41,435 DEBUG [M:0;3302f0f507bd:36429 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T00:14:41,436 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T00:14:41,436 DEBUG [M:0;3302f0f507bd:36429 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T00:14:41,436 DEBUG [master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.large.0-1731024874225 {}] cleaner.HFileCleaner(306): Exit Thread[master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.large.0-1731024874225,5,FailOnTimeoutGroup] 2024-11-08T00:14:41,436 DEBUG [master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.small.0-1731024874226 {}] cleaner.HFileCleaner(306): Exit Thread[master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.small.0-1731024874226,5,FailOnTimeoutGroup] 2024-11-08T00:14:41,436 INFO [M:0;3302f0f507bd:36429 {}] hbase.ChoreService(370): Chore service for: master/3302f0f507bd:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T00:14:41,436 INFO [M:0;3302f0f507bd:36429 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:14:41,436 DEBUG [M:0;3302f0f507bd:36429 {}] master.HMaster(1795): Stopping service threads 2024-11-08T00:14:41,436 INFO [M:0;3302f0f507bd:36429 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T00:14:41,436 INFO [M:0;3302f0f507bd:36429 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:14:41,436 INFO [M:0;3302f0f507bd:36429 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:14:41,437 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T00:14:41,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T00:14:41,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:14:41,446 DEBUG [M:0;3302f0f507bd:36429 {}] zookeeper.ZKUtil(347): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T00:14:41,446 WARN [M:0;3302f0f507bd:36429 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T00:14:41,446 INFO [M:0;3302f0f507bd:36429 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/.lastflushedseqids 2024-11-08T00:14:41,449 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T00:14:41,449 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T00:14:41,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-08T00:14:41,451 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-08T00:14:41,452 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T00:14:41,452 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T00:14:41,452 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T00:14:41,452 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T00:14:41,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741870_1046 (size=1560) 2024-11-08T00:14:41,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741870_1046 (size=1560) 2024-11-08T00:14:41,461 INFO [M:0;3302f0f507bd:36429 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:14:41,461 INFO [M:0;3302f0f507bd:36429 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T00:14:41,462 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T00:14:41,462 INFO [M:0;3302f0f507bd:36429 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:14:41,462 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:14:41,462 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T00:14:41,462 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:14:41,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741870_1046 (size=1560) 2024-11-08T00:14:41,462 INFO [M:0;3302f0f507bd:36429 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.07 KB heapSize=301.48 KB 2024-11-08T00:14:41,480 DEBUG [M:0;3302f0f507bd:36429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7cac16f925724825893f363926b0bf72 is 82, key is hbase:meta,,1/info:regioninfo/1731024879260/Put/seqid=0 2024-11-08T00:14:41,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741871_1047 (size=6071) 2024-11-08T00:14:41,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741871_1047 (size=6071) 2024-11-08T00:14:41,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741871_1047 (size=6071) 2024-11-08T00:14:41,488 INFO [M:0;3302f0f507bd:36429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7cac16f925724825893f363926b0bf72 2024-11-08T00:14:41,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:14:41,512 INFO [RS:0;3302f0f507bd:38371 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:14:41,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38371-0x10117cb57f40001, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:14:41,512 INFO [RS:0;3302f0f507bd:38371 {}] regionserver.HRegionServer(1031): Exiting; stopping=3302f0f507bd,38371,1731024872767; zookeeper connection closed. 2024-11-08T00:14:41,512 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73821d22 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73821d22 2024-11-08T00:14:41,517 DEBUG [M:0;3302f0f507bd:36429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e9c45251b774ba380164f9384918725 is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731024881122/Put/seqid=0 2024-11-08T00:14:41,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:14:41,522 INFO [RS:2;3302f0f507bd:36139 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:14:41,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36139-0x10117cb57f40003, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:14:41,522 INFO [RS:2;3302f0f507bd:36139 {}] regionserver.HRegionServer(1031): Exiting; stopping=3302f0f507bd,36139,1731024872925; zookeeper connection closed. 2024-11-08T00:14:41,523 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@45033ceb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@45033ceb 2024-11-08T00:14:41,523 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T00:14:41,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741872_1048 (size=12799) 2024-11-08T00:14:41,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741872_1048 (size=12799) 2024-11-08T00:14:41,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741872_1048 (size=12799) 2024-11-08T00:14:41,525 INFO [M:0;3302f0f507bd:36429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=246.80 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e9c45251b774ba380164f9384918725 2024-11-08T00:14:41,546 DEBUG [M:0;3302f0f507bd:36429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9dcb1247e452463ea263b2ddf284f9ab is 69, key is 3302f0f507bd,36139,1731024872925/rs:state/1731024874375/Put/seqid=0 2024-11-08T00:14:41,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741873_1049 (size=5442) 2024-11-08T00:14:41,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741873_1049 (size=5442) 2024-11-08T00:14:41,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741873_1049 (size=5442) 2024-11-08T00:14:41,554 INFO [M:0;3302f0f507bd:36429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9dcb1247e452463ea263b2ddf284f9ab 2024-11-08T00:14:41,559 INFO [M:0;3302f0f507bd:36429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9dcb1247e452463ea263b2ddf284f9ab 2024-11-08T00:14:41,575 DEBUG [M:0;3302f0f507bd:36429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ec7a572b9c594deda5c4a6df73e5a1c8 is 52, key is load_balancer_on/state:d/1731024877957/Put/seqid=0 2024-11-08T00:14:41,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741874_1050 (size=5058) 2024-11-08T00:14:41,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741874_1050 (size=5058) 2024-11-08T00:14:41,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36957 is added to blk_1073741874_1050 (size=5058) 2024-11-08T00:14:41,583 INFO [M:0;3302f0f507bd:36429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ec7a572b9c594deda5c4a6df73e5a1c8 2024-11-08T00:14:41,591 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7cac16f925724825893f363926b0bf72 as hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7cac16f925724825893f363926b0bf72 2024-11-08T00:14:41,599 INFO [M:0;3302f0f507bd:36429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7cac16f925724825893f363926b0bf72, entries=14, sequenceid=569, filesize=5.9 K 2024-11-08T00:14:41,600 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e9c45251b774ba380164f9384918725 as hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7e9c45251b774ba380164f9384918725 2024-11-08T00:14:41,611 INFO [M:0;3302f0f507bd:36429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7e9c45251b774ba380164f9384918725, entries=79, sequenceid=569, filesize=12.5 K 2024-11-08T00:14:41,613 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9dcb1247e452463ea263b2ddf284f9ab as hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9dcb1247e452463ea263b2ddf284f9ab 2024-11-08T00:14:41,621 INFO [M:0;3302f0f507bd:36429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9dcb1247e452463ea263b2ddf284f9ab 2024-11-08T00:14:41,621 INFO [M:0;3302f0f507bd:36429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9dcb1247e452463ea263b2ddf284f9ab, entries=3, sequenceid=569, filesize=5.3 K 2024-11-08T00:14:41,622 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ec7a572b9c594deda5c4a6df73e5a1c8 as hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ec7a572b9c594deda5c4a6df73e5a1c8 2024-11-08T00:14:41,630 INFO [M:0;3302f0f507bd:36429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44875/user/jenkins/test-data/77b53e35-6e91-cdf0-645f-a04f2a48aa20/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ec7a572b9c594deda5c4a6df73e5a1c8, entries=1, sequenceid=569, filesize=4.9 K 2024-11-08T00:14:41,633 INFO [M:0;3302f0f507bd:36429 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.07 KB/254028, heapSize ~301.41 KB/308648, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 171ms, sequenceid=569, compaction requested=false 2024-11-08T00:14:41,635 INFO [M:0;3302f0f507bd:36429 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:14:41,635 DEBUG [M:0;3302f0f507bd:36429 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731024881461Disabling compacts and flushes for region at 1731024881461Disabling writes for close at 1731024881462 (+1 ms)Obtaining lock to block concurrent updates at 1731024881462Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731024881462Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254028, getHeapSize=308648, getOffHeapSize=0, getCellsCount=708 at 1731024881462Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731024881463 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731024881463Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731024881480 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731024881480Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731024881495 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731024881517 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731024881517Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731024881532 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731024881545 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731024881545Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731024881559 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731024881574 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731024881574Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17378bca: reopening flushed file at 1731024881590 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47e807e7: reopening flushed file at 1731024881599 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26ad3591: reopening flushed file at 1731024881612 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@84e4acc: reopening flushed file at 1731024881621 (+9 ms)Finished flush of dataSize ~248.07 KB/254028, heapSize ~301.41 KB/308648, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 171ms, sequenceid=569, compaction requested=false at 1731024881633 (+12 ms)Writing region close event to WAL at 1731024881635 (+2 ms)Closed at 1731024881635 2024-11-08T00:14:41,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36391 is added to blk_1073741830_1006 (size=292022) 2024-11-08T00:14:41,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44189 is added to blk_1073741830_1006 (size=292022) 2024-11-08T00:14:41,639 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:14:41,639 INFO [M:0;3302f0f507bd:36429 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T00:14:41,639 INFO [M:0;3302f0f507bd:36429 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36429 2024-11-08T00:14:41,640 INFO [M:0;3302f0f507bd:36429 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:14:41,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:14:41,811 INFO [M:0;3302f0f507bd:36429 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:14:41,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36429-0x10117cb57f40000, quorum=127.0.0.1:54358, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:14:41,814 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T00:14:41,816 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T00:14:41,816 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T00:14:41,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T00:14:41,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.log.dir/,STOPPED} 2024-11-08T00:14:41,818 WARN [BP-76941553-172.17.0.3-1731024865284 heartbeating to localhost/127.0.0.1:44875 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T00:14:41,818 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T00:14:41,818 WARN [BP-76941553-172.17.0.3-1731024865284 heartbeating to localhost/127.0.0.1:44875 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-76941553-172.17.0.3-1731024865284 (Datanode Uuid 89c99946-75ec-4aaf-bc46-cfe5f0ddb9fc) service to localhost/127.0.0.1:44875 2024-11-08T00:14:41,819 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T00:14:41,820 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data5/current/BP-76941553-172.17.0.3-1731024865284 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T00:14:41,820 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data6/current/BP-76941553-172.17.0.3-1731024865284 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T00:14:41,821 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T00:14:41,823 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T00:14:41,824 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T00:14:41,824 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T00:14:41,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T00:14:41,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.log.dir/,STOPPED} 2024-11-08T00:14:41,825 WARN [BP-76941553-172.17.0.3-1731024865284 heartbeating to localhost/127.0.0.1:44875 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T00:14:41,825 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T00:14:41,825 WARN [BP-76941553-172.17.0.3-1731024865284 heartbeating to localhost/127.0.0.1:44875 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-76941553-172.17.0.3-1731024865284 (Datanode Uuid 82969024-0613-43db-8af9-2144424c9c8e) service to localhost/127.0.0.1:44875 2024-11-08T00:14:41,825 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T00:14:41,826 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data3/current/BP-76941553-172.17.0.3-1731024865284 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T00:14:41,826 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data4/current/BP-76941553-172.17.0.3-1731024865284 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T00:14:41,826 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T00:14:41,833 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T00:14:41,833 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T00:14:41,833 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T00:14:41,833 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T00:14:41,834 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.log.dir/,STOPPED} 2024-11-08T00:14:41,835 WARN [BP-76941553-172.17.0.3-1731024865284 heartbeating to localhost/127.0.0.1:44875 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T00:14:41,835 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T00:14:41,835 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T00:14:41,835 WARN [BP-76941553-172.17.0.3-1731024865284 heartbeating to localhost/127.0.0.1:44875 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-76941553-172.17.0.3-1731024865284 (Datanode Uuid d75300ff-4353-4e3c-98b5-6111bd901f5b) service to localhost/127.0.0.1:44875 2024-11-08T00:14:41,836 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data1/current/BP-76941553-172.17.0.3-1731024865284 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T00:14:41,836 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/cluster_06b526fe-5bb5-a249-aeab-5fb8b82ca8f2/data/data2/current/BP-76941553-172.17.0.3-1731024865284 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T00:14:41,836 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T00:14:41,844 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T00:14:41,845 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T00:14:41,845 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T00:14:41,845 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T00:14:41,845 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/463e9217-7eb9-3382-a7b4-01eeb5c23d4a/hadoop.log.dir/,STOPPED} 2024-11-08T00:14:41,854 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T00:14:41,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down