2024-11-24 23:13:18,184 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-24 23:13:18,197 main DEBUG Took 0.010619 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 23:13:18,197 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 23:13:18,198 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 23:13:18,199 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 23:13:18,200 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,207 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 23:13:18,221 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,223 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,223 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,224 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,225 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,226 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,227 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,227 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,228 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,229 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,229 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,230 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,230 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,231 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,231 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,232 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,232 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,232 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,233 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,233 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,234 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,234 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,235 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 23:13:18,235 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,236 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 23:13:18,238 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 23:13:18,239 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 23:13:18,241 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 23:13:18,242 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 23:13:18,243 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 23:13:18,244 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 23:13:18,251 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 23:13:18,254 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 23:13:18,256 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 23:13:18,256 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 23:13:18,257 main DEBUG createAppenders(={Console}) 2024-11-24 23:13:18,258 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-24 23:13:18,258 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-24 23:13:18,258 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-24 23:13:18,259 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 23:13:18,259 main DEBUG OutputStream closed 2024-11-24 23:13:18,259 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 23:13:18,260 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 23:13:18,260 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-24 23:13:18,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 23:13:18,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 23:13:18,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 23:13:18,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 23:13:18,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 23:13:18,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 23:13:18,351 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 23:13:18,351 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 23:13:18,352 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 23:13:18,352 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 23:13:18,353 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 23:13:18,353 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 23:13:18,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 23:13:18,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 23:13:18,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 23:13:18,355 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 23:13:18,355 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 23:13:18,356 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 23:13:18,359 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 23:13:18,359 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-24 23:13:18,360 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 23:13:18,360 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-24T23:13:18,375 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-24 23:13:18,378 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 23:13:18,378 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T23:13:18,625 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9 2024-11-24T23:13:18,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T23:13:18,656 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800, deleteOnExit=true 2024-11-24T23:13:18,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T23:13:18,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/test.cache.data in system properties and HBase conf 2024-11-24T23:13:18,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T23:13:18,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.log.dir in system properties and HBase conf 2024-11-24T23:13:18,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T23:13:18,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T23:13:18,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T23:13:18,750 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T23:13:18,842 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T23:13:18,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T23:13:18,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T23:13:18,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T23:13:18,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T23:13:18,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T23:13:18,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T23:13:18,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T23:13:18,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T23:13:18,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T23:13:18,851 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/nfs.dump.dir in system properties and HBase conf 2024-11-24T23:13:18,851 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/java.io.tmpdir in system properties and HBase conf 2024-11-24T23:13:18,851 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T23:13:18,852 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T23:13:18,852 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T23:13:19,996 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T23:13:20,110 INFO [Time-limited test {}] log.Log(170): Logging initialized @2616ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T23:13:20,227 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T23:13:20,321 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T23:13:20,358 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T23:13:20,359 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T23:13:20,361 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T23:13:20,379 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T23:13:20,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.log.dir/,AVAILABLE} 2024-11-24T23:13:20,384 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T23:13:20,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/java.io.tmpdir/jetty-localhost-43907-hadoop-hdfs-3_4_1-tests_jar-_-any-8892190797710407377/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T23:13:20,601 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:43907} 2024-11-24T23:13:20,602 INFO [Time-limited test {}] server.Server(415): Started @3109ms 2024-11-24T23:13:21,385 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T23:13:21,396 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T23:13:21,397 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T23:13:21,397 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T23:13:21,398 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T23:13:21,399 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.log.dir/,AVAILABLE} 2024-11-24T23:13:21,400 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T23:13:21,529 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/java.io.tmpdir/jetty-localhost-44661-hadoop-hdfs-3_4_1-tests_jar-_-any-14702815255078248110/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T23:13:21,530 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:44661} 2024-11-24T23:13:21,530 INFO [Time-limited test {}] server.Server(415): Started @4037ms 2024-11-24T23:13:21,611 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T23:13:21,786 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T23:13:21,793 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T23:13:21,804 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T23:13:21,805 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T23:13:21,805 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T23:13:21,808 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.log.dir/,AVAILABLE} 2024-11-24T23:13:21,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T23:13:21,950 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/java.io.tmpdir/jetty-localhost-34821-hadoop-hdfs-3_4_1-tests_jar-_-any-10159300558141975345/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T23:13:21,951 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:34821} 2024-11-24T23:13:21,951 INFO [Time-limited test {}] server.Server(415): Started @4458ms 2024-11-24T23:13:21,953 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T23:13:22,001 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T23:13:22,008 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T23:13:22,009 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T23:13:22,009 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T23:13:22,009 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T23:13:22,010 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.log.dir/,AVAILABLE} 2024-11-24T23:13:22,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T23:13:22,108 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/java.io.tmpdir/jetty-localhost-45715-hadoop-hdfs-3_4_1-tests_jar-_-any-7295126095007338306/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T23:13:22,109 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:45715} 2024-11-24T23:13:22,109 INFO [Time-limited test {}] server.Server(415): Started @4617ms 2024-11-24T23:13:22,111 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T23:13:23,700 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data4/current/BP-1799454338-172.17.0.3-1732489999444/current, will proceed with Du for space computation calculation, 2024-11-24T23:13:23,700 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data3/current/BP-1799454338-172.17.0.3-1732489999444/current, will proceed with Du for space computation calculation, 2024-11-24T23:13:23,700 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data2/current/BP-1799454338-172.17.0.3-1732489999444/current, will proceed with Du for space computation calculation, 2024-11-24T23:13:23,700 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data1/current/BP-1799454338-172.17.0.3-1732489999444/current, will proceed with Du for space computation calculation, 2024-11-24T23:13:23,731 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T23:13:23,731 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T23:13:23,737 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data5/current/BP-1799454338-172.17.0.3-1732489999444/current, will proceed with Du for space computation calculation, 2024-11-24T23:13:23,738 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data6/current/BP-1799454338-172.17.0.3-1732489999444/current, will proceed with Du for space computation calculation, 2024-11-24T23:13:23,762 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T23:13:23,781 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c67ad9087f11f60 with lease ID 0xba20fd8e7e26811a: Processing first storage report for DS-be315e5e-9a06-443e-884f-6714183c1b21 from datanode DatanodeRegistration(127.0.0.1:40001, datanodeUuid=4c22eff2-9f74-488e-8ab8-03f0164c7ebb, infoPort=46315, infoSecurePort=0, ipcPort=33093, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444) 2024-11-24T23:13:23,782 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c67ad9087f11f60 with lease ID 0xba20fd8e7e26811a: from storage DS-be315e5e-9a06-443e-884f-6714183c1b21 node DatanodeRegistration(127.0.0.1:40001, datanodeUuid=4c22eff2-9f74-488e-8ab8-03f0164c7ebb, infoPort=46315, infoSecurePort=0, ipcPort=33093, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T23:13:23,782 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbdbd40afa67d1180 with lease ID 0xba20fd8e7e26811c: Processing first storage report for DS-a0b0d19b-e93b-4f83-a9ec-e0dbbf30d950 from datanode DatanodeRegistration(127.0.0.1:39131, datanodeUuid=6c3b17cf-6945-4e2b-804c-c4d98c9b57ef, infoPort=41507, infoSecurePort=0, ipcPort=33747, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444) 2024-11-24T23:13:23,782 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbdbd40afa67d1180 with lease ID 0xba20fd8e7e26811c: from storage DS-a0b0d19b-e93b-4f83-a9ec-e0dbbf30d950 node DatanodeRegistration(127.0.0.1:39131, datanodeUuid=6c3b17cf-6945-4e2b-804c-c4d98c9b57ef, infoPort=41507, infoSecurePort=0, ipcPort=33747, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T23:13:23,783 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84330de42e02d488 with lease ID 0xba20fd8e7e26811b: Processing first storage report for DS-2b455eb7-85ba-41ff-ac8b-19e614915e5f from datanode DatanodeRegistration(127.0.0.1:44853, datanodeUuid=2245d66d-029d-4ff6-9903-409dd1be8822, infoPort=44785, infoSecurePort=0, ipcPort=37595, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444) 2024-11-24T23:13:23,783 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84330de42e02d488 with lease ID 0xba20fd8e7e26811b: from storage DS-2b455eb7-85ba-41ff-ac8b-19e614915e5f node DatanodeRegistration(127.0.0.1:44853, datanodeUuid=2245d66d-029d-4ff6-9903-409dd1be8822, infoPort=44785, infoSecurePort=0, ipcPort=37595, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T23:13:23,783 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c67ad9087f11f60 with lease ID 0xba20fd8e7e26811a: Processing first storage report for DS-aa4e5996-6ea7-4469-9b5e-45289d35b259 from datanode DatanodeRegistration(127.0.0.1:40001, datanodeUuid=4c22eff2-9f74-488e-8ab8-03f0164c7ebb, infoPort=46315, infoSecurePort=0, ipcPort=33093, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444) 2024-11-24T23:13:23,783 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c67ad9087f11f60 with lease ID 0xba20fd8e7e26811a: from storage DS-aa4e5996-6ea7-4469-9b5e-45289d35b259 node DatanodeRegistration(127.0.0.1:40001, datanodeUuid=4c22eff2-9f74-488e-8ab8-03f0164c7ebb, infoPort=46315, infoSecurePort=0, ipcPort=33093, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T23:13:23,784 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbdbd40afa67d1180 with lease ID 0xba20fd8e7e26811c: Processing first storage report for DS-f3b04df1-a409-4bef-afa1-9b3c0b15e663 from datanode DatanodeRegistration(127.0.0.1:39131, datanodeUuid=6c3b17cf-6945-4e2b-804c-c4d98c9b57ef, infoPort=41507, infoSecurePort=0, ipcPort=33747, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444) 2024-11-24T23:13:23,784 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbdbd40afa67d1180 with lease ID 0xba20fd8e7e26811c: from storage DS-f3b04df1-a409-4bef-afa1-9b3c0b15e663 node DatanodeRegistration(127.0.0.1:39131, datanodeUuid=6c3b17cf-6945-4e2b-804c-c4d98c9b57ef, infoPort=41507, infoSecurePort=0, ipcPort=33747, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T23:13:23,784 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84330de42e02d488 with lease ID 0xba20fd8e7e26811b: Processing first storage report for DS-508567b7-d44c-4282-ad7e-445475cabb16 from datanode DatanodeRegistration(127.0.0.1:44853, datanodeUuid=2245d66d-029d-4ff6-9903-409dd1be8822, infoPort=44785, infoSecurePort=0, ipcPort=37595, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444) 2024-11-24T23:13:23,784 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84330de42e02d488 with lease ID 0xba20fd8e7e26811b: from storage DS-508567b7-d44c-4282-ad7e-445475cabb16 node DatanodeRegistration(127.0.0.1:44853, datanodeUuid=2245d66d-029d-4ff6-9903-409dd1be8822, infoPort=44785, infoSecurePort=0, ipcPort=37595, storageInfo=lv=-57;cid=testClusterID;nsid=1526659100;c=1732489999444), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T23:13:23,869 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9 2024-11-24T23:13:23,937 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/zookeeper_0, clientPort=60106, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T23:13:23,946 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60106 2024-11-24T23:13:23,956 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:23,959 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:24,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741825_1001 (size=7) 2024-11-24T23:13:24,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741825_1001 (size=7) 2024-11-24T23:13:24,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741825_1001 (size=7) 2024-11-24T23:13:24,589 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 with version=8 2024-11-24T23:13:24,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/hbase-staging 2024-11-24T23:13:24,667 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T23:13:24,876 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ba7f32532130:0 server-side Connection retries=45 2024-11-24T23:13:24,885 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:24,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:24,890 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T23:13:24,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:24,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T23:13:25,019 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T23:13:25,075 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T23:13:25,083 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T23:13:25,087 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T23:13:25,110 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2327 (auto-detected) 2024-11-24T23:13:25,111 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-24T23:13:25,129 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34129 2024-11-24T23:13:25,149 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34129 connecting to ZooKeeper ensemble=127.0.0.1:60106 2024-11-24T23:13:25,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:341290x0, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T23:13:25,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34129-0x1016f1f81790000 connected 2024-11-24T23:13:25,369 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:25,373 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:25,384 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T23:13:25,389 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4, hbase.cluster.distributed=false 2024-11-24T23:13:25,414 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T23:13:25,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34129 2024-11-24T23:13:25,419 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34129 2024-11-24T23:13:25,419 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34129 2024-11-24T23:13:25,420 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34129 2024-11-24T23:13:25,420 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34129 2024-11-24T23:13:25,521 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45 2024-11-24T23:13:25,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:25,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:25,523 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T23:13:25,523 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:25,523 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T23:13:25,525 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T23:13:25,528 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T23:13:25,529 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45393 2024-11-24T23:13:25,531 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45393 connecting to ZooKeeper ensemble=127.0.0.1:60106 2024-11-24T23:13:25,532 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:25,534 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:25,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453930x0, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T23:13:25,547 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453930x0, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T23:13:25,547 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45393-0x1016f1f81790001 connected 2024-11-24T23:13:25,552 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T23:13:25,559 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T23:13:25,561 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T23:13:25,566 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T23:13:25,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45393 2024-11-24T23:13:25,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45393 2024-11-24T23:13:25,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45393 2024-11-24T23:13:25,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45393 2024-11-24T23:13:25,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45393 2024-11-24T23:13:25,585 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45 2024-11-24T23:13:25,585 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:25,585 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:25,586 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T23:13:25,586 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:25,586 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T23:13:25,586 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T23:13:25,586 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T23:13:25,587 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37739 2024-11-24T23:13:25,589 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37739 connecting to ZooKeeper ensemble=127.0.0.1:60106 2024-11-24T23:13:25,590 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:25,592 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:25,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377390x0, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T23:13:25,605 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:377390x0, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T23:13:25,606 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37739-0x1016f1f81790002 connected 2024-11-24T23:13:25,606 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T23:13:25,608 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T23:13:25,609 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T23:13:25,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T23:13:25,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37739 2024-11-24T23:13:25,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37739 2024-11-24T23:13:25,612 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37739 2024-11-24T23:13:25,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37739 2024-11-24T23:13:25,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37739 2024-11-24T23:13:25,632 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45 2024-11-24T23:13:25,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:25,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:25,633 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T23:13:25,633 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T23:13:25,633 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T23:13:25,633 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T23:13:25,634 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T23:13:25,634 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36885 2024-11-24T23:13:25,637 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36885 connecting to ZooKeeper ensemble=127.0.0.1:60106 2024-11-24T23:13:25,639 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:25,642 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:25,658 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368850x0, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T23:13:25,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:368850x0, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T23:13:25,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36885-0x1016f1f81790003 connected 2024-11-24T23:13:25,659 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T23:13:25,661 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T23:13:25,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T23:13:25,664 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T23:13:25,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36885 2024-11-24T23:13:25,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36885 2024-11-24T23:13:25,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36885 2024-11-24T23:13:25,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36885 2024-11-24T23:13:25,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36885 2024-11-24T23:13:25,685 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba7f32532130:34129 2024-11-24T23:13:25,686 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba7f32532130,34129,1732490004717 2024-11-24T23:13:25,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T23:13:25,704 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T23:13:25,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T23:13:25,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T23:13:25,707 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba7f32532130,34129,1732490004717 2024-11-24T23:13:25,735 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T23:13:25,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T23:13:25,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:25,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T23:13:25,736 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:25,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:25,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:25,737 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T23:13:25,739 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba7f32532130,34129,1732490004717 from backup master directory 2024-11-24T23:13:25,752 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T23:13:25,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T23:13:25,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba7f32532130,34129,1732490004717 2024-11-24T23:13:25,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T23:13:25,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T23:13:25,753 WARN [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T23:13:25,753 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba7f32532130,34129,1732490004717 2024-11-24T23:13:25,756 INFO [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T23:13:25,758 INFO [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T23:13:25,816 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/hbase.id] with ID: 090cdb7e-9e49-4db4-b43f-aaf5a68d12f0 2024-11-24T23:13:25,817 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/.tmp/hbase.id 2024-11-24T23:13:25,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741826_1002 (size=42) 2024-11-24T23:13:25,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741826_1002 (size=42) 2024-11-24T23:13:25,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741826_1002 (size=42) 2024-11-24T23:13:25,833 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/.tmp/hbase.id]:[hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/hbase.id] 2024-11-24T23:13:25,877 INFO [master/ba7f32532130:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T23:13:25,881 INFO [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T23:13:25,900 INFO [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-24T23:13:25,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:25,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:25,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:25,915 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:25,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741827_1003 (size=196) 2024-11-24T23:13:25,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741827_1003 (size=196) 2024-11-24T23:13:25,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741827_1003 (size=196) 2024-11-24T23:13:25,949 INFO [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T23:13:25,951 INFO [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T23:13:25,965 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T23:13:25,969 INFO [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T23:13:26,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741828_1004 (size=1189) 2024-11-24T23:13:26,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741828_1004 (size=1189) 2024-11-24T23:13:26,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741828_1004 (size=1189) 2024-11-24T23:13:26,023 INFO [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store 2024-11-24T23:13:26,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741829_1005 (size=34) 2024-11-24T23:13:26,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741829_1005 (size=34) 2024-11-24T23:13:26,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741829_1005 (size=34) 2024-11-24T23:13:26,051 INFO [master/ba7f32532130:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T23:13:26,054 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:26,055 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T23:13:26,056 INFO [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T23:13:26,056 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T23:13:26,057 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T23:13:26,057 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T23:13:26,058 INFO [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T23:13:26,059 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732490006055Disabling compacts and flushes for region at 1732490006055Disabling writes for close at 1732490006057 (+2 ms)Writing region close event to WAL at 1732490006058 (+1 ms)Closed at 1732490006058 2024-11-24T23:13:26,061 WARN [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/.initializing 2024-11-24T23:13:26,061 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/WALs/ba7f32532130,34129,1732490004717 2024-11-24T23:13:26,070 INFO [master/ba7f32532130:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T23:13:26,083 INFO [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C34129%2C1732490004717, suffix=, logDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/WALs/ba7f32532130,34129,1732490004717, archiveDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/oldWALs, maxLogs=10 2024-11-24T23:13:26,106 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/WALs/ba7f32532130,34129,1732490004717/ba7f32532130%2C34129%2C1732490004717.1732490006088, exclude list is [], retry=0 2024-11-24T23:13:26,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39131,DS-a0b0d19b-e93b-4f83-a9ec-e0dbbf30d950,DISK] 2024-11-24T23:13:26,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40001,DS-be315e5e-9a06-443e-884f-6714183c1b21,DISK] 2024-11-24T23:13:26,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44853,DS-2b455eb7-85ba-41ff-ac8b-19e614915e5f,DISK] 2024-11-24T23:13:26,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T23:13:26,169 INFO [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/WALs/ba7f32532130,34129,1732490004717/ba7f32532130%2C34129%2C1732490004717.1732490006088 2024-11-24T23:13:26,170 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41507:41507),(127.0.0.1/127.0.0.1:46315:46315),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-24T23:13:26,170 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T23:13:26,170 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:26,173 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,174 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,212 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T23:13:26,243 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:26,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:26,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T23:13:26,250 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:26,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:26,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T23:13:26,255 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:26,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:26,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,259 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T23:13:26,260 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:26,261 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:26,261 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,266 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,267 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,274 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,275 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,278 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T23:13:26,282 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T23:13:26,288 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:26,289 INFO [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59848537, jitterRate=-0.10818730294704437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T23:13:26,298 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732490006185Initializing all the Stores at 1732490006188 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490006188Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490006189 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490006189Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490006189Cleaning up temporary data from old regions at 1732490006275 (+86 ms)Region opened successfully at 1732490006298 (+23 ms) 2024-11-24T23:13:26,299 INFO [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T23:13:26,333 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db02e33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0 2024-11-24T23:13:26,360 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T23:13:26,369 INFO [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:26,369 INFO [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:26,372 INFO [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:26,373 INFO [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T23:13:26,377 INFO [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T23:13:26,377 INFO [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T23:13:26,400 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T23:13:26,408 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T23:13:26,458 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T23:13:26,462 INFO [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T23:13:26,465 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T23:13:26,478 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T23:13:26,480 INFO [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T23:13:26,483 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T23:13:26,493 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T23:13:26,495 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T23:13:26,504 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T23:13:26,522 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T23:13:26,530 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T23:13:26,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T23:13:26,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T23:13:26,541 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T23:13:26,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T23:13:26,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,542 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,546 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba7f32532130,34129,1732490004717, sessionid=0x1016f1f81790000, setting cluster-up flag (Was=false) 2024-11-24T23:13:26,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,578 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,609 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T23:13:26,614 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,34129,1732490004717 2024-11-24T23:13:26,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,641 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:26,672 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T23:13:26,674 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,34129,1732490004717 2024-11-24T23:13:26,682 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T23:13:26,748 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:26,758 INFO [master/ba7f32532130:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T23:13:26,763 INFO [master/ba7f32532130:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T23:13:26,769 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba7f32532130,34129,1732490004717 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T23:13:26,774 INFO [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer(746): ClusterId : 090cdb7e-9e49-4db4-b43f-aaf5a68d12f0 2024-11-24T23:13:26,774 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(746): ClusterId : 090cdb7e-9e49-4db4-b43f-aaf5a68d12f0 2024-11-24T23:13:26,774 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(746): ClusterId : 090cdb7e-9e49-4db4-b43f-aaf5a68d12f0 2024-11-24T23:13:26,776 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5 2024-11-24T23:13:26,776 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5 2024-11-24T23:13:26,777 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5 2024-11-24T23:13:26,777 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T23:13:26,777 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T23:13:26,777 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T23:13:26,777 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5 2024-11-24T23:13:26,777 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba7f32532130:0, corePoolSize=10, maxPoolSize=10 2024-11-24T23:13:26,777 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:26,777 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba7f32532130:0, corePoolSize=2, maxPoolSize=2 2024-11-24T23:13:26,777 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:26,778 INFO [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490036778 2024-11-24T23:13:26,779 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T23:13:26,780 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T23:13:26,782 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:26,782 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T23:13:26,783 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T23:13:26,784 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T23:13:26,784 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T23:13:26,784 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T23:13:26,785 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:26,787 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T23:13:26,787 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:26,787 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T23:13:26,788 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T23:13:26,788 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T23:13:26,790 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T23:13:26,790 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T23:13:26,790 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T23:13:26,790 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T23:13:26,790 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T23:13:26,790 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T23:13:26,792 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T23:13:26,793 INFO [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T23:13:26,794 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732490006794,5,FailOnTimeoutGroup] 2024-11-24T23:13:26,795 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732490006794,5,FailOnTimeoutGroup] 2024-11-24T23:13:26,795 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:26,795 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T23:13:26,796 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:26,797 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:26,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741831_1007 (size=1321) 2024-11-24T23:13:26,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741831_1007 (size=1321) 2024-11-24T23:13:26,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741831_1007 (size=1321) 2024-11-24T23:13:26,809 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T23:13:26,809 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:26,814 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T23:13:26,814 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T23:13:26,814 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T23:13:26,814 DEBUG [RS:2;ba7f32532130:36885 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f52e9d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0 2024-11-24T23:13:26,814 DEBUG [RS:1;ba7f32532130:37739 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d95d0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0 2024-11-24T23:13:26,814 DEBUG [RS:0;ba7f32532130:45393 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fe658d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0 2024-11-24T23:13:26,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741832_1008 (size=32) 2024-11-24T23:13:26,830 DEBUG [RS:2;ba7f32532130:36885 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ba7f32532130:36885 2024-11-24T23:13:26,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741832_1008 (size=32) 2024-11-24T23:13:26,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741832_1008 (size=32) 2024-11-24T23:13:26,832 DEBUG [RS:0;ba7f32532130:45393 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba7f32532130:45393 2024-11-24T23:13:26,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:26,834 INFO [RS:2;ba7f32532130:36885 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T23:13:26,834 INFO [RS:0;ba7f32532130:45393 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T23:13:26,834 INFO [RS:0;ba7f32532130:45393 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T23:13:26,834 INFO [RS:2;ba7f32532130:36885 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T23:13:26,834 DEBUG [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T23:13:26,834 DEBUG [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T23:13:26,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T23:13:26,837 INFO [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,34129,1732490004717 with port=36885, startcode=1732490005631 2024-11-24T23:13:26,837 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,34129,1732490004717 with port=45393, startcode=1732490005487 2024-11-24T23:13:26,838 DEBUG [RS:1;ba7f32532130:37739 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ba7f32532130:37739 2024-11-24T23:13:26,838 INFO [RS:1;ba7f32532130:37739 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T23:13:26,838 INFO [RS:1;ba7f32532130:37739 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T23:13:26,838 DEBUG [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T23:13:26,839 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,34129,1732490004717 with port=37739, startcode=1732490005584 2024-11-24T23:13:26,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T23:13:26,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:26,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:26,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T23:13:26,845 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T23:13:26,845 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:26,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:26,847 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T23:13:26,850 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T23:13:26,850 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:26,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:26,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T23:13:26,853 DEBUG [RS:2;ba7f32532130:36885 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T23:13:26,853 DEBUG [RS:1;ba7f32532130:37739 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T23:13:26,853 DEBUG [RS:0;ba7f32532130:45393 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T23:13:26,855 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T23:13:26,855 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:26,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:26,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T23:13:26,858 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740 2024-11-24T23:13:26,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740 2024-11-24T23:13:26,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T23:13:26,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T23:13:26,864 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T23:13:26,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T23:13:26,878 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:26,879 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72628471, jitterRate=0.08224855363368988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T23:13:26,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732490006833Initializing all the Stores at 1732490006835 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490006835Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490006836 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490006836Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490006836Cleaning up temporary data from old regions at 1732490006863 (+27 ms)Region opened successfully at 1732490006884 (+21 ms) 2024-11-24T23:13:26,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T23:13:26,885 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T23:13:26,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T23:13:26,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T23:13:26,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T23:13:26,887 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T23:13:26,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732490006885Disabling compacts and flushes for region at 1732490006885Disabling writes for close at 1732490006886 (+1 ms)Writing region close event to WAL at 1732490006887 (+1 ms)Closed at 1732490006887 2024-11-24T23:13:26,892 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:26,892 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T23:13:26,894 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41585, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T23:13:26,894 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57921, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T23:13:26,894 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40675, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T23:13:26,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T23:13:26,902 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,37739,1732490005584 2024-11-24T23:13:26,904 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,37739,1732490005584 2024-11-24T23:13:26,913 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:26,918 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T23:13:26,918 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,45393,1732490005487 2024-11-24T23:13:26,919 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,45393,1732490005487 2024-11-24T23:13:26,923 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,36885,1732490005631 2024-11-24T23:13:26,923 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,36885,1732490005631 2024-11-24T23:13:26,924 DEBUG [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:26,925 DEBUG [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41705 2024-11-24T23:13:26,925 DEBUG [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T23:13:26,925 DEBUG [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:26,926 DEBUG [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41705 2024-11-24T23:13:26,926 DEBUG [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T23:13:26,929 DEBUG [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:26,929 DEBUG [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41705 2024-11-24T23:13:26,929 DEBUG [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T23:13:26,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T23:13:26,988 DEBUG [RS:1;ba7f32532130:37739 {}] zookeeper.ZKUtil(111): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,37739,1732490005584 2024-11-24T23:13:26,988 DEBUG [RS:0;ba7f32532130:45393 {}] zookeeper.ZKUtil(111): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,45393,1732490005487 2024-11-24T23:13:26,988 WARN [RS:1;ba7f32532130:37739 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T23:13:26,989 WARN [RS:0;ba7f32532130:45393 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T23:13:26,989 INFO [RS:1;ba7f32532130:37739 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T23:13:26,989 INFO [RS:0;ba7f32532130:45393 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T23:13:26,989 DEBUG [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,37739,1732490005584 2024-11-24T23:13:26,989 DEBUG [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,45393,1732490005487 2024-11-24T23:13:26,989 DEBUG [RS:2;ba7f32532130:36885 {}] zookeeper.ZKUtil(111): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,36885,1732490005631 2024-11-24T23:13:26,989 WARN [RS:2;ba7f32532130:36885 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T23:13:26,989 INFO [RS:2;ba7f32532130:36885 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T23:13:26,990 DEBUG [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631 2024-11-24T23:13:26,991 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,36885,1732490005631] 2024-11-24T23:13:26,991 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,37739,1732490005584] 2024-11-24T23:13:26,992 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,45393,1732490005487] 2024-11-24T23:13:27,023 INFO [RS:1;ba7f32532130:37739 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T23:13:27,023 INFO [RS:2;ba7f32532130:36885 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T23:13:27,023 INFO [RS:0;ba7f32532130:45393 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T23:13:27,044 INFO [RS:1;ba7f32532130:37739 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T23:13:27,044 INFO [RS:2;ba7f32532130:36885 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T23:13:27,044 INFO [RS:0;ba7f32532130:45393 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T23:13:27,050 INFO [RS:2;ba7f32532130:36885 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T23:13:27,050 INFO [RS:0;ba7f32532130:45393 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T23:13:27,050 INFO [RS:1;ba7f32532130:37739 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T23:13:27,050 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,050 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,050 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,051 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T23:13:27,052 INFO [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T23:13:27,052 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T23:13:27,058 INFO [RS:0;ba7f32532130:45393 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T23:13:27,058 INFO [RS:2;ba7f32532130:36885 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T23:13:27,058 INFO [RS:1;ba7f32532130:37739 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T23:13:27,060 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,060 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,060 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,060 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,060 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,060 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,060 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,060 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,060 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2 2024-11-24T23:13:27,061 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2 2024-11-24T23:13:27,061 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2 2024-11-24T23:13:27,061 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,061 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3 2024-11-24T23:13:27,062 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:0;ba7f32532130:45393 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3 2024-11-24T23:13:27,062 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3 2024-11-24T23:13:27,062 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1 2024-11-24T23:13:27,062 DEBUG [RS:2;ba7f32532130:36885 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3 2024-11-24T23:13:27,062 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3 2024-11-24T23:13:27,062 DEBUG [RS:1;ba7f32532130:37739 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3 2024-11-24T23:13:27,069 WARN [ba7f32532130:34129 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T23:13:27,073 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,074 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,074 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,074 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,074 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,074 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,36885,1732490005631-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T23:13:27,075 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,075 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,075 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,075 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,075 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,075 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,075 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,075 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,076 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,37739,1732490005584-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T23:13:27,076 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,076 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,076 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45393,1732490005487-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T23:13:27,103 INFO [RS:2;ba7f32532130:36885 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T23:13:27,103 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T23:13:27,106 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,36885,1732490005631-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,106 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45393,1732490005487-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,106 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,106 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,107 INFO [RS:2;ba7f32532130:36885 {}] regionserver.Replication(171): ba7f32532130,36885,1732490005631 started 2024-11-24T23:13:27,107 INFO [RS:0;ba7f32532130:45393 {}] regionserver.Replication(171): ba7f32532130,45393,1732490005487 started 2024-11-24T23:13:27,128 INFO [RS:2;ba7f32532130:36885 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,129 INFO [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,36885,1732490005631, RpcServer on ba7f32532130/172.17.0.3:36885, sessionid=0x1016f1f81790003 2024-11-24T23:13:27,130 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T23:13:27,130 DEBUG [RS:2;ba7f32532130:36885 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,36885,1732490005631 2024-11-24T23:13:27,131 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,36885,1732490005631' 2024-11-24T23:13:27,131 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,131 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T23:13:27,131 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,45393,1732490005487, RpcServer on ba7f32532130/172.17.0.3:45393, sessionid=0x1016f1f81790001 2024-11-24T23:13:27,131 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T23:13:27,131 DEBUG [RS:0;ba7f32532130:45393 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,45393,1732490005487 2024-11-24T23:13:27,131 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,45393,1732490005487' 2024-11-24T23:13:27,131 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T23:13:27,132 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T23:13:27,132 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T23:13:27,133 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T23:13:27,133 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T23:13:27,133 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T23:13:27,133 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T23:13:27,133 DEBUG [RS:0;ba7f32532130:45393 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,45393,1732490005487 2024-11-24T23:13:27,133 DEBUG [RS:2;ba7f32532130:36885 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,36885,1732490005631 2024-11-24T23:13:27,133 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,45393,1732490005487' 2024-11-24T23:13:27,133 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,36885,1732490005631' 2024-11-24T23:13:27,133 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T23:13:27,133 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T23:13:27,134 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T23:13:27,134 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T23:13:27,134 DEBUG [RS:2;ba7f32532130:36885 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T23:13:27,134 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T23:13:27,134 INFO [RS:2;ba7f32532130:36885 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T23:13:27,134 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,37739,1732490005584-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,135 INFO [RS:2;ba7f32532130:36885 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T23:13:27,135 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,135 INFO [RS:1;ba7f32532130:37739 {}] regionserver.Replication(171): ba7f32532130,37739,1732490005584 started 2024-11-24T23:13:27,135 DEBUG [RS:0;ba7f32532130:45393 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T23:13:27,135 INFO [RS:0;ba7f32532130:45393 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T23:13:27,135 INFO [RS:0;ba7f32532130:45393 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T23:13:27,159 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,159 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,37739,1732490005584, RpcServer on ba7f32532130/172.17.0.3:37739, sessionid=0x1016f1f81790002 2024-11-24T23:13:27,160 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T23:13:27,160 DEBUG [RS:1;ba7f32532130:37739 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,37739,1732490005584 2024-11-24T23:13:27,160 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,37739,1732490005584' 2024-11-24T23:13:27,160 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T23:13:27,161 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T23:13:27,162 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T23:13:27,162 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T23:13:27,162 DEBUG [RS:1;ba7f32532130:37739 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,37739,1732490005584 2024-11-24T23:13:27,162 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,37739,1732490005584' 2024-11-24T23:13:27,162 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T23:13:27,163 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T23:13:27,164 DEBUG [RS:1;ba7f32532130:37739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T23:13:27,164 INFO [RS:1;ba7f32532130:37739 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T23:13:27,164 INFO [RS:1;ba7f32532130:37739 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T23:13:27,240 INFO [RS:2;ba7f32532130:36885 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T23:13:27,240 INFO [RS:0;ba7f32532130:45393 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T23:13:27,243 INFO [RS:2;ba7f32532130:36885 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C36885%2C1732490005631, suffix=, logDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631, archiveDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs, maxLogs=32 2024-11-24T23:13:27,243 INFO [RS:0;ba7f32532130:45393 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C45393%2C1732490005487, suffix=, logDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,45393,1732490005487, archiveDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs, maxLogs=32 2024-11-24T23:13:27,261 DEBUG [RS:2;ba7f32532130:36885 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631/ba7f32532130%2C36885%2C1732490005631.1732490007246, exclude list is [], retry=0 2024-11-24T23:13:27,261 DEBUG [RS:0;ba7f32532130:45393 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,45393,1732490005487/ba7f32532130%2C45393%2C1732490005487.1732490007246, exclude list is [], retry=0 2024-11-24T23:13:27,265 INFO [RS:1;ba7f32532130:37739 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T23:13:27,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40001,DS-be315e5e-9a06-443e-884f-6714183c1b21,DISK] 2024-11-24T23:13:27,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40001,DS-be315e5e-9a06-443e-884f-6714183c1b21,DISK] 2024-11-24T23:13:27,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44853,DS-2b455eb7-85ba-41ff-ac8b-19e614915e5f,DISK] 2024-11-24T23:13:27,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44853,DS-2b455eb7-85ba-41ff-ac8b-19e614915e5f,DISK] 2024-11-24T23:13:27,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39131,DS-a0b0d19b-e93b-4f83-a9ec-e0dbbf30d950,DISK] 2024-11-24T23:13:27,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39131,DS-a0b0d19b-e93b-4f83-a9ec-e0dbbf30d950,DISK] 2024-11-24T23:13:27,268 INFO [RS:1;ba7f32532130:37739 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C37739%2C1732490005584, suffix=, logDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,37739,1732490005584, archiveDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs, maxLogs=32 2024-11-24T23:13:27,271 INFO [RS:0;ba7f32532130:45393 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,45393,1732490005487/ba7f32532130%2C45393%2C1732490005487.1732490007246 2024-11-24T23:13:27,271 INFO [RS:2;ba7f32532130:36885 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631/ba7f32532130%2C36885%2C1732490005631.1732490007246 2024-11-24T23:13:27,271 DEBUG [RS:2;ba7f32532130:36885 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46315:46315),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:41507:41507)] 2024-11-24T23:13:27,271 DEBUG [RS:0;ba7f32532130:45393 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46315:46315),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:41507:41507)] 2024-11-24T23:13:27,284 DEBUG [RS:1;ba7f32532130:37739 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,37739,1732490005584/ba7f32532130%2C37739%2C1732490005584.1732490007271, exclude list is [], retry=0 2024-11-24T23:13:27,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39131,DS-a0b0d19b-e93b-4f83-a9ec-e0dbbf30d950,DISK] 2024-11-24T23:13:27,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44853,DS-2b455eb7-85ba-41ff-ac8b-19e614915e5f,DISK] 2024-11-24T23:13:27,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40001,DS-be315e5e-9a06-443e-884f-6714183c1b21,DISK] 2024-11-24T23:13:27,293 INFO [RS:1;ba7f32532130:37739 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,37739,1732490005584/ba7f32532130%2C37739%2C1732490005584.1732490007271 2024-11-24T23:13:27,293 DEBUG [RS:1;ba7f32532130:37739 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41507:41507),(127.0.0.1/127.0.0.1:46315:46315),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-24T23:13:27,322 DEBUG [ba7f32532130:34129 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T23:13:27,332 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(204): Hosts are {ba7f32532130=0} racks are {/default-rack=0} 2024-11-24T23:13:27,339 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T23:13:27,339 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T23:13:27,339 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T23:13:27,339 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T23:13:27,339 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T23:13:27,339 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T23:13:27,339 INFO [ba7f32532130:34129 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T23:13:27,339 INFO [ba7f32532130:34129 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T23:13:27,339 INFO [ba7f32532130:34129 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T23:13:27,339 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T23:13:27,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:27,352 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,36885,1732490005631, state=OPENING 2024-11-24T23:13:27,404 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T23:13:27,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:27,415 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:27,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:27,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:27,416 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:27,416 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:27,416 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:27,416 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:27,418 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:27,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,36885,1732490005631}] 2024-11-24T23:13:27,600 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T23:13:27,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58123, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T23:13:27,615 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T23:13:27,615 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T23:13:27,616 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T23:13:27,619 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C36885%2C1732490005631.meta, suffix=.meta, logDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631, archiveDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs, maxLogs=32 2024-11-24T23:13:27,633 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, exclude list is [], retry=0 2024-11-24T23:13:27,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40001,DS-be315e5e-9a06-443e-884f-6714183c1b21,DISK] 2024-11-24T23:13:27,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39131,DS-a0b0d19b-e93b-4f83-a9ec-e0dbbf30d950,DISK] 2024-11-24T23:13:27,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44853,DS-2b455eb7-85ba-41ff-ac8b-19e614915e5f,DISK] 2024-11-24T23:13:27,640 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta 2024-11-24T23:13:27,643 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46315:46315),(127.0.0.1/127.0.0.1:41507:41507),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-24T23:13:27,643 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T23:13:27,645 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T23:13:27,648 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T23:13:27,653 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T23:13:27,657 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T23:13:27,658 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:27,658 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T23:13:27,658 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T23:13:27,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T23:13:27,663 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T23:13:27,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:27,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:27,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T23:13:27,666 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T23:13:27,666 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:27,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:27,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T23:13:27,668 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T23:13:27,668 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:27,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:27,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T23:13:27,671 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T23:13:27,671 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:27,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:27,672 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T23:13:27,673 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740 2024-11-24T23:13:27,676 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740 2024-11-24T23:13:27,678 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T23:13:27,678 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T23:13:27,679 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T23:13:27,682 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T23:13:27,684 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72961273, jitterRate=0.08720768988132477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T23:13:27,684 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T23:13:27,686 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732490007659Writing region info on filesystem at 1732490007659Initializing all the Stores at 1732490007661 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490007661Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490007661Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490007661Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490007661Cleaning up temporary data from old regions at 1732490007678 (+17 ms)Running coprocessor post-open hooks at 1732490007684 (+6 ms)Region opened successfully at 1732490007686 (+2 ms) 2024-11-24T23:13:27,694 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732490007589 2024-11-24T23:13:27,705 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T23:13:27,706 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T23:13:27,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:27,710 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,36885,1732490005631, state=OPEN 2024-11-24T23:13:27,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:27,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:27,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:27,720 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:27,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:27,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:27,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:27,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:27,721 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba7f32532130,36885,1732490005631 2024-11-24T23:13:27,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T23:13:27,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,36885,1732490005631 in 300 msec 2024-11-24T23:13:27,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T23:13:27,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 830 msec 2024-11-24T23:13:27,737 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:27,737 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T23:13:27,758 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T23:13:27,759 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1] 2024-11-24T23:13:27,779 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T23:13:27,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53289, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T23:13:27,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0920 sec 2024-11-24T23:13:27,803 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732490007803, completionTime=-1 2024-11-24T23:13:27,807 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T23:13:27,808 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:27,861 INFO [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T23:13:27,862 INFO [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490067861 2024-11-24T23:13:27,862 INFO [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490127862 2024-11-24T23:13:27,862 INFO [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 54 msec 2024-11-24T23:13:27,864 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:27,877 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,34129,1732490004717-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,877 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,34129,1732490004717-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,877 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,34129,1732490004717-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,880 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba7f32532130:34129, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,880 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,881 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,889 DEBUG [master/ba7f32532130:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T23:13:27,918 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.164sec 2024-11-24T23:13:27,919 INFO [master/ba7f32532130:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T23:13:27,920 INFO [master/ba7f32532130:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T23:13:27,921 INFO [master/ba7f32532130:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T23:13:27,922 INFO [master/ba7f32532130:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T23:13:27,922 INFO [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T23:13:27,923 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,34129,1732490004717-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T23:13:27,923 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,34129,1732490004717-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T23:13:27,928 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T23:13:27,929 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T23:13:27,929 INFO [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,34129,1732490004717-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T23:13:27,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39f0836e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T23:13:27,992 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T23:13:27,992 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T23:13:27,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba7f32532130,34129,-1 for getting cluster id 2024-11-24T23:13:27,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T23:13:28,006 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '090cdb7e-9e49-4db4-b43f-aaf5a68d12f0' 2024-11-24T23:13:28,008 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T23:13:28,008 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "090cdb7e-9e49-4db4-b43f-aaf5a68d12f0" 2024-11-24T23:13:28,009 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc9ce1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T23:13:28,009 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba7f32532130,34129,-1] 2024-11-24T23:13:28,011 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T23:13:28,013 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:28,014 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T23:13:28,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41704ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T23:13:28,017 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T23:13:28,024 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1] 2024-11-24T23:13:28,025 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T23:13:28,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T23:13:28,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba7f32532130,34129,1732490004717 2024-11-24T23:13:28,051 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T23:13:28,056 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ba7f32532130,34129,1732490004717 2024-11-24T23:13:28,059 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@62315734 2024-11-24T23:13:28,061 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T23:13:28,063 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T23:13:28,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T23:13:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:28,082 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T23:13:28,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-24T23:13:28,084 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,087 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T23:13:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T23:13:28,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741837_1013 (size=358) 2024-11-24T23:13:28,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741837_1013 (size=358) 2024-11-24T23:13:28,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741837_1013 (size=358) 2024-11-24T23:13:28,106 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => afc068dc7ba93a54eff038a8f06c8542, NAME => 'test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,106 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b4c5d46f0872640f4da48af2d12d1131, NAME => 'test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,107 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4120d4bb2e1399a50983e420a0eeded8, NAME => 'test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741838_1014 (size=45) 2024-11-24T23:13:28,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741838_1014 (size=45) 2024-11-24T23:13:28,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741838_1014 (size=45) 2024-11-24T23:13:28,128 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,128 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b4c5d46f0872640f4da48af2d12d1131, disabling compactions & flushes 2024-11-24T23:13:28,128 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:28,128 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:28,128 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. after waiting 0 ms 2024-11-24T23:13:28,128 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:28,128 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:28,128 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b4c5d46f0872640f4da48af2d12d1131: Waiting for close lock at 1732490008128Disabling compacts and flushes for region at 1732490008128Disabling writes for close at 1732490008128Writing region close event to WAL at 1732490008128Closed at 1732490008128 2024-11-24T23:13:28,129 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c8ee88c2c22e3a06388845a14a86f9b2, NAME => 'test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741839_1015 (size=45) 2024-11-24T23:13:28,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741840_1016 (size=42) 2024-11-24T23:13:28,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741840_1016 (size=42) 2024-11-24T23:13:28,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741839_1015 (size=45) 2024-11-24T23:13:28,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741839_1015 (size=45) 2024-11-24T23:13:28,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741840_1016 (size=42) 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing afc068dc7ba93a54eff038a8f06c8542, disabling compactions & flushes 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4120d4bb2e1399a50983e420a0eeded8, disabling compactions & flushes 2024-11-24T23:13:28,143 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:28,143 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. after waiting 0 ms 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. after waiting 0 ms 2024-11-24T23:13:28,143 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for afc068dc7ba93a54eff038a8f06c8542: Waiting for close lock at 1732490008143Disabling compacts and flushes for region at 1732490008143Disabling writes for close at 1732490008143Writing region close event to WAL at 1732490008143Closed at 1732490008143 2024-11-24T23:13:28,143 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:28,143 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4120d4bb2e1399a50983e420a0eeded8: Waiting for close lock at 1732490008143Disabling compacts and flushes for region at 1732490008143Disabling writes for close at 1732490008143Writing region close event to WAL at 1732490008143Closed at 1732490008143 2024-11-24T23:13:28,144 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ab12bee6921e08663b5265ccf589a9d, NAME => 'test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,145 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => eb705a3a04d42fd23798b48207c112de, NAME => 'test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741841_1017 (size=45) 2024-11-24T23:13:28,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741841_1017 (size=45) 2024-11-24T23:13:28,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741841_1017 (size=45) 2024-11-24T23:13:28,175 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,175 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c8ee88c2c22e3a06388845a14a86f9b2, disabling compactions & flushes 2024-11-24T23:13:28,175 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:28,175 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:28,175 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. after waiting 0 ms 2024-11-24T23:13:28,176 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:28,176 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:28,176 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c8ee88c2c22e3a06388845a14a86f9b2: Waiting for close lock at 1732490008175Disabling compacts and flushes for region at 1732490008175Disabling writes for close at 1732490008175Writing region close event to WAL at 1732490008176 (+1 ms)Closed at 1732490008176 2024-11-24T23:13:28,177 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 79f30edc5579a1fb86c63410b3b6d59a, NAME => 'test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741842_1018 (size=45) 2024-11-24T23:13:28,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741842_1018 (size=45) 2024-11-24T23:13:28,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741842_1018 (size=45) 2024-11-24T23:13:28,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 0ab12bee6921e08663b5265ccf589a9d, disabling compactions & flushes 2024-11-24T23:13:28,186 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:28,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:28,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. after waiting 0 ms 2024-11-24T23:13:28,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:28,186 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:28,186 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ab12bee6921e08663b5265ccf589a9d: Waiting for close lock at 1732490008186Disabling compacts and flushes for region at 1732490008186Disabling writes for close at 1732490008186Writing region close event to WAL at 1732490008186Closed at 1732490008186 2024-11-24T23:13:28,187 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 740a2018f9a8f09b78987d38b376567b, NAME => 'test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741843_1019 (size=45) 2024-11-24T23:13:28,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741843_1019 (size=45) 2024-11-24T23:13:28,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741843_1019 (size=45) 2024-11-24T23:13:28,193 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing eb705a3a04d42fd23798b48207c112de, disabling compactions & flushes 2024-11-24T23:13:28,194 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:28,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:28,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. after waiting 0 ms 2024-11-24T23:13:28,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:28,194 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:28,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for eb705a3a04d42fd23798b48207c112de: Waiting for close lock at 1732490008194Disabling compacts and flushes for region at 1732490008194Disabling writes for close at 1732490008194Writing region close event to WAL at 1732490008194Closed at 1732490008194 2024-11-24T23:13:28,195 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6e72e703f9026a43993a24ead31db485, NAME => 'test,iii,1732490008065.6e72e703f9026a43993a24ead31db485.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T23:13:28,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741844_1020 (size=45) 2024-11-24T23:13:28,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741844_1020 (size=45) 2024-11-24T23:13:28,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741845_1021 (size=45) 2024-11-24T23:13:28,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741845_1021 (size=45) 2024-11-24T23:13:28,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741845_1021 (size=45) 2024-11-24T23:13:28,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741844_1020 (size=45) 2024-11-24T23:13:28,211 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,211 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 740a2018f9a8f09b78987d38b376567b, disabling compactions & flushes 2024-11-24T23:13:28,211 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:28,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:28,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. after waiting 0 ms 2024-11-24T23:13:28,212 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:28,212 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:28,212 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 79f30edc5579a1fb86c63410b3b6d59a, disabling compactions & flushes 2024-11-24T23:13:28,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 740a2018f9a8f09b78987d38b376567b: Waiting for close lock at 1732490008211Disabling compacts and flushes for region at 1732490008211Disabling writes for close at 1732490008212 (+1 ms)Writing region close event to WAL at 1732490008212Closed at 1732490008212 2024-11-24T23:13:28,212 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:28,212 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:28,212 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. after waiting 0 ms 2024-11-24T23:13:28,212 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 97ef64c446508710b8b41414b50b97d8, NAME => 'test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,213 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:28,213 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:28,213 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 79f30edc5579a1fb86c63410b3b6d59a: Waiting for close lock at 1732490008212Disabling compacts and flushes for region at 1732490008212Disabling writes for close at 1732490008213 (+1 ms)Writing region close event to WAL at 1732490008213Closed at 1732490008213 2024-11-24T23:13:28,213 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => df65ea81e19f0ed6cb718601a302e9ef, NAME => 'test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741846_1022 (size=45) 2024-11-24T23:13:28,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741846_1022 (size=45) 2024-11-24T23:13:28,221 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1732490008065.6e72e703f9026a43993a24ead31db485.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,221 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6e72e703f9026a43993a24ead31db485, disabling compactions & flushes 2024-11-24T23:13:28,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741846_1022 (size=45) 2024-11-24T23:13:28,221 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:28,221 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:28,222 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. after waiting 0 ms 2024-11-24T23:13:28,222 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:28,222 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:28,222 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6e72e703f9026a43993a24ead31db485: Waiting for close lock at 1732490008221Disabling compacts and flushes for region at 1732490008221Disabling writes for close at 1732490008222 (+1 ms)Writing region close event to WAL at 1732490008222Closed at 1732490008222 2024-11-24T23:13:28,223 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7af30662d67e218e5eed0cf39272545b, NAME => 'test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741848_1024 (size=45) 2024-11-24T23:13:28,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741848_1024 (size=45) 2024-11-24T23:13:28,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741848_1024 (size=45) 2024-11-24T23:13:28,238 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,238 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing df65ea81e19f0ed6cb718601a302e9ef, disabling compactions & flushes 2024-11-24T23:13:28,238 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:28,238 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:28,238 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. after waiting 0 ms 2024-11-24T23:13:28,238 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:28,238 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:28,238 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for df65ea81e19f0ed6cb718601a302e9ef: Waiting for close lock at 1732490008238Disabling compacts and flushes for region at 1732490008238Disabling writes for close at 1732490008238Writing region close event to WAL at 1732490008238Closed at 1732490008238 2024-11-24T23:13:28,239 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b32141aa52f6862fa21af187e3ac3b72, NAME => 'test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741847_1023 (size=45) 2024-11-24T23:13:28,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741847_1023 (size=45) 2024-11-24T23:13:28,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741847_1023 (size=45) 2024-11-24T23:13:28,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 97ef64c446508710b8b41414b50b97d8, disabling compactions & flushes 2024-11-24T23:13:28,248 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:28,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:28,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. after waiting 0 ms 2024-11-24T23:13:28,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:28,248 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:28,249 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 97ef64c446508710b8b41414b50b97d8: Waiting for close lock at 1732490008248Disabling compacts and flushes for region at 1732490008248Disabling writes for close at 1732490008248Writing region close event to WAL at 1732490008248Closed at 1732490008248 2024-11-24T23:13:28,250 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 69fc34e6caf2036ee015474372306228, NAME => 'test,nnn,1732490008065.69fc34e6caf2036ee015474372306228.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741849_1025 (size=45) 2024-11-24T23:13:28,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741849_1025 (size=45) 2024-11-24T23:13:28,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741849_1025 (size=45) 2024-11-24T23:13:28,264 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,264 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7af30662d67e218e5eed0cf39272545b, disabling compactions & flushes 2024-11-24T23:13:28,265 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:28,265 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:28,265 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. after waiting 0 ms 2024-11-24T23:13:28,265 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:28,265 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:28,265 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7af30662d67e218e5eed0cf39272545b: Waiting for close lock at 1732490008264Disabling compacts and flushes for region at 1732490008264Disabling writes for close at 1732490008265 (+1 ms)Writing region close event to WAL at 1732490008265Closed at 1732490008265 2024-11-24T23:13:28,266 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 792edfc226c30220c15d3c2a8c9ffa60, NAME => 'test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741850_1026 (size=45) 2024-11-24T23:13:28,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741850_1026 (size=45) 2024-11-24T23:13:28,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741850_1026 (size=45) 2024-11-24T23:13:28,270 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,270 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b32141aa52f6862fa21af187e3ac3b72, disabling compactions & flushes 2024-11-24T23:13:28,270 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:28,270 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:28,271 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. after waiting 0 ms 2024-11-24T23:13:28,271 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:28,271 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:28,271 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b32141aa52f6862fa21af187e3ac3b72: Waiting for close lock at 1732490008270Disabling compacts and flushes for region at 1732490008270Disabling writes for close at 1732490008271 (+1 ms)Writing region close event to WAL at 1732490008271Closed at 1732490008271 2024-11-24T23:13:28,272 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6a817676ac0dff05fc27e6ec9f8e6c89, NAME => 'test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741851_1027 (size=45) 2024-11-24T23:13:28,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741851_1027 (size=45) 2024-11-24T23:13:28,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741851_1027 (size=45) 2024-11-24T23:13:28,278 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1732490008065.69fc34e6caf2036ee015474372306228.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,278 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 69fc34e6caf2036ee015474372306228, disabling compactions & flushes 2024-11-24T23:13:28,278 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:28,278 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:28,278 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. after waiting 0 ms 2024-11-24T23:13:28,278 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:28,278 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:28,278 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 69fc34e6caf2036ee015474372306228: Waiting for close lock at 1732490008278Disabling compacts and flushes for region at 1732490008278Disabling writes for close at 1732490008278Writing region close event to WAL at 1732490008278Closed at 1732490008278 2024-11-24T23:13:28,279 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7a1389df947c85301aa553a546616498, NAME => 'test,qqq,1732490008065.7a1389df947c85301aa553a546616498.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741852_1028 (size=45) 2024-11-24T23:13:28,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741852_1028 (size=45) 2024-11-24T23:13:28,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741852_1028 (size=45) 2024-11-24T23:13:28,285 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,285 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 792edfc226c30220c15d3c2a8c9ffa60, disabling compactions & flushes 2024-11-24T23:13:28,285 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:28,285 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:28,285 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. after waiting 0 ms 2024-11-24T23:13:28,285 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:28,285 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:28,285 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 792edfc226c30220c15d3c2a8c9ffa60: Waiting for close lock at 1732490008285Disabling compacts and flushes for region at 1732490008285Disabling writes for close at 1732490008285Writing region close event to WAL at 1732490008285Closed at 1732490008285 2024-11-24T23:13:28,286 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1244e0d7b7b5b8b2d792af82faee1da6, NAME => 'test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741853_1029 (size=45) 2024-11-24T23:13:28,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741853_1029 (size=45) 2024-11-24T23:13:28,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741853_1029 (size=45) 2024-11-24T23:13:28,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6a817676ac0dff05fc27e6ec9f8e6c89, disabling compactions & flushes 2024-11-24T23:13:28,290 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:28,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:28,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. after waiting 0 ms 2024-11-24T23:13:28,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:28,290 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:28,290 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6a817676ac0dff05fc27e6ec9f8e6c89: Waiting for close lock at 1732490008290Disabling compacts and flushes for region at 1732490008290Disabling writes for close at 1732490008290Writing region close event to WAL at 1732490008290Closed at 1732490008290 2024-11-24T23:13:28,291 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6874a4cd5ca2cb048061687443fd87c9, NAME => 'test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741854_1030 (size=45) 2024-11-24T23:13:28,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741854_1030 (size=45) 2024-11-24T23:13:28,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741854_1030 (size=45) 2024-11-24T23:13:28,301 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1732490008065.7a1389df947c85301aa553a546616498.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,301 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7a1389df947c85301aa553a546616498, disabling compactions & flushes 2024-11-24T23:13:28,301 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:28,301 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:28,301 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732490008065.7a1389df947c85301aa553a546616498. after waiting 0 ms 2024-11-24T23:13:28,302 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:28,302 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:28,302 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7a1389df947c85301aa553a546616498: Waiting for close lock at 1732490008301Disabling compacts and flushes for region at 1732490008301Disabling writes for close at 1732490008302 (+1 ms)Writing region close event to WAL at 1732490008302Closed at 1732490008302 2024-11-24T23:13:28,302 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c50f7d0b8194465f687d68a15e4d9d13, NAME => 'test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741855_1031 (size=45) 2024-11-24T23:13:28,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741855_1031 (size=45) 2024-11-24T23:13:28,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741855_1031 (size=45) 2024-11-24T23:13:28,314 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,314 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 1244e0d7b7b5b8b2d792af82faee1da6, disabling compactions & flushes 2024-11-24T23:13:28,314 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:28,314 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:28,314 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. after waiting 0 ms 2024-11-24T23:13:28,314 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:28,314 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:28,314 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1244e0d7b7b5b8b2d792af82faee1da6: Waiting for close lock at 1732490008314Disabling compacts and flushes for region at 1732490008314Disabling writes for close at 1732490008314Writing region close event to WAL at 1732490008314Closed at 1732490008314 2024-11-24T23:13:28,315 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f6455d829ffb35ae0e811982e8b77549, NAME => 'test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741856_1032 (size=45) 2024-11-24T23:13:28,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741856_1032 (size=45) 2024-11-24T23:13:28,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741856_1032 (size=45) 2024-11-24T23:13:28,318 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,319 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6874a4cd5ca2cb048061687443fd87c9, disabling compactions & flushes 2024-11-24T23:13:28,319 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:28,319 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:28,319 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. after waiting 0 ms 2024-11-24T23:13:28,319 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:28,319 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:28,319 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6874a4cd5ca2cb048061687443fd87c9: Waiting for close lock at 1732490008319Disabling compacts and flushes for region at 1732490008319Disabling writes for close at 1732490008319Writing region close event to WAL at 1732490008319Closed at 1732490008319 2024-11-24T23:13:28,320 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a5362f4dec21f673244ee873440d1779, NAME => 'test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741857_1033 (size=45) 2024-11-24T23:13:28,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741857_1033 (size=45) 2024-11-24T23:13:28,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741857_1033 (size=45) 2024-11-24T23:13:28,325 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,325 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c50f7d0b8194465f687d68a15e4d9d13, disabling compactions & flushes 2024-11-24T23:13:28,325 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:28,325 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:28,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. after waiting 0 ms 2024-11-24T23:13:28,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:28,326 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:28,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c50f7d0b8194465f687d68a15e4d9d13: Waiting for close lock at 1732490008325Disabling compacts and flushes for region at 1732490008325Disabling writes for close at 1732490008326 (+1 ms)Writing region close event to WAL at 1732490008326Closed at 1732490008326 2024-11-24T23:13:28,327 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 64743c3cc4326247d7ff0a96361c7e49, NAME => 'test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741858_1034 (size=45) 2024-11-24T23:13:28,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741858_1034 (size=45) 2024-11-24T23:13:28,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741858_1034 (size=45) 2024-11-24T23:13:28,342 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,342 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f6455d829ffb35ae0e811982e8b77549, disabling compactions & flushes 2024-11-24T23:13:28,342 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:28,342 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:28,342 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. after waiting 0 ms 2024-11-24T23:13:28,342 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:28,342 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:28,343 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f6455d829ffb35ae0e811982e8b77549: Waiting for close lock at 1732490008342Disabling compacts and flushes for region at 1732490008342Disabling writes for close at 1732490008342Writing region close event to WAL at 1732490008342Closed at 1732490008342 2024-11-24T23:13:28,343 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 90985da3d485ab013ba37f15b715a5d0, NAME => 'test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741859_1035 (size=45) 2024-11-24T23:13:28,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741859_1035 (size=45) 2024-11-24T23:13:28,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741859_1035 (size=45) 2024-11-24T23:13:28,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741860_1036 (size=45) 2024-11-24T23:13:28,346 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741860_1036 (size=45) 2024-11-24T23:13:28,346 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a5362f4dec21f673244ee873440d1779, disabling compactions & flushes 2024-11-24T23:13:28,346 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:28,346 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:28,346 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. after waiting 0 ms 2024-11-24T23:13:28,346 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:28,346 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:28,347 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a5362f4dec21f673244ee873440d1779: Waiting for close lock at 1732490008346Disabling compacts and flushes for region at 1732490008346Disabling writes for close at 1732490008346Writing region close event to WAL at 1732490008346Closed at 1732490008346 2024-11-24T23:13:28,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741860_1036 (size=45) 2024-11-24T23:13:28,347 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2f0e42305404ba59b5243b2bf8292cf0, NAME => 'test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,348 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,348 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 64743c3cc4326247d7ff0a96361c7e49, disabling compactions & flushes 2024-11-24T23:13:28,348 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:28,348 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:28,348 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. after waiting 0 ms 2024-11-24T23:13:28,348 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:28,348 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:28,349 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 64743c3cc4326247d7ff0a96361c7e49: Waiting for close lock at 1732490008348Disabling compacts and flushes for region at 1732490008348Disabling writes for close at 1732490008348Writing region close event to WAL at 1732490008348Closed at 1732490008348 2024-11-24T23:13:28,349 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => adb73b3b244f40e15cbf9254ee22925b, NAME => 'test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4 2024-11-24T23:13:28,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741861_1037 (size=45) 2024-11-24T23:13:28,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741861_1037 (size=45) 2024-11-24T23:13:28,358 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741861_1037 (size=45) 2024-11-24T23:13:28,358 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 90985da3d485ab013ba37f15b715a5d0, disabling compactions & flushes 2024-11-24T23:13:28,358 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:28,358 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:28,358 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. after waiting 0 ms 2024-11-24T23:13:28,359 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:28,359 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:28,359 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 90985da3d485ab013ba37f15b715a5d0: Waiting for close lock at 1732490008358Disabling compacts and flushes for region at 1732490008358Disabling writes for close at 1732490008358Writing region close event to WAL at 1732490008359 (+1 ms)Closed at 1732490008359 2024-11-24T23:13:28,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741863_1039 (size=42) 2024-11-24T23:13:28,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741863_1039 (size=42) 2024-11-24T23:13:28,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741863_1039 (size=42) 2024-11-24T23:13:28,365 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,365 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing adb73b3b244f40e15cbf9254ee22925b, disabling compactions & flushes 2024-11-24T23:13:28,365 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:28,365 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:28,365 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. after waiting 0 ms 2024-11-24T23:13:28,366 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:28,366 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:28,366 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for adb73b3b244f40e15cbf9254ee22925b: Waiting for close lock at 1732490008365Disabling compacts and flushes for region at 1732490008365Disabling writes for close at 1732490008366 (+1 ms)Writing region close event to WAL at 1732490008366Closed at 1732490008366 2024-11-24T23:13:28,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741862_1038 (size=45) 2024-11-24T23:13:28,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741862_1038 (size=45) 2024-11-24T23:13:28,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741862_1038 (size=45) 2024-11-24T23:13:28,372 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,372 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2f0e42305404ba59b5243b2bf8292cf0, disabling compactions & flushes 2024-11-24T23:13:28,372 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:28,372 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:28,372 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. after waiting 0 ms 2024-11-24T23:13:28,372 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:28,372 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:28,372 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2f0e42305404ba59b5243b2bf8292cf0: Waiting for close lock at 1732490008372Disabling compacts and flushes for region at 1732490008372Disabling writes for close at 1732490008372Writing region close event to WAL at 1732490008372Closed at 1732490008372 2024-11-24T23:13:28,375 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T23:13:28,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732490008065.6e72e703f9026a43993a24ead31db485.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732490008065.69fc34e6caf2036ee015474372306228.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732490008065.7a1389df947c85301aa553a546616498.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,384 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,384 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,384 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,384 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,384 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732490008375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732490008375"}]},"ts":"1732490008375"} 2024-11-24T23:13:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T23:13:28,425 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-24T23:13:28,428 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T23:13:28,431 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732490008428"}]},"ts":"1732490008428"} 2024-11-24T23:13:28,435 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-24T23:13:28,436 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {ba7f32532130=0} racks are {/default-rack=0} 2024-11-24T23:13:28,454 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T23:13:28,454 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T23:13:28,454 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T23:13:28,454 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T23:13:28,454 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T23:13:28,454 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T23:13:28,454 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T23:13:28,454 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T23:13:28,454 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T23:13:28,454 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T23:13:28,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=afc068dc7ba93a54eff038a8f06c8542, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b4c5d46f0872640f4da48af2d12d1131, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8ee88c2c22e3a06388845a14a86f9b2, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0ab12bee6921e08663b5265ccf589a9d, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79f30edc5579a1fb86c63410b3b6d59a, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=740a2018f9a8f09b78987d38b376567b, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97ef64c446508710b8b41414b50b97d8, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df65ea81e19f0ed6cb718601a302e9ef, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b32141aa52f6862fa21af187e3ac3b72, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=69fc34e6caf2036ee015474372306228, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a817676ac0dff05fc27e6ec9f8e6c89, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7a1389df947c85301aa553a546616498, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6874a4cd5ca2cb048061687443fd87c9, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c50f7d0b8194465f687d68a15e4d9d13, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5362f4dec21f673244ee873440d1779, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64743c3cc4326247d7ff0a96361c7e49, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f0e42305404ba59b5243b2bf8292cf0, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=adb73b3b244f40e15cbf9254ee22925b, ASSIGN}] 2024-11-24T23:13:28,467 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=adb73b3b244f40e15cbf9254ee22925b, ASSIGN 2024-11-24T23:13:28,468 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=adb73b3b244f40e15cbf9254ee22925b, ASSIGN; state=OFFLINE, location=ba7f32532130,37739,1732490005584; forceNewPlan=false, retain=false 2024-11-24T23:13:28,470 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2f0e42305404ba59b5243b2bf8292cf0, ASSIGN 2024-11-24T23:13:28,471 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2f0e42305404ba59b5243b2bf8292cf0, ASSIGN; state=OFFLINE, location=ba7f32532130,45393,1732490005487; forceNewPlan=false, retain=false 2024-11-24T23:13:28,472 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:28,473 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN; state=OFFLINE, location=ba7f32532130,36885,1732490005631; forceNewPlan=false, retain=false 2024-11-24T23:13:28,475 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64743c3cc4326247d7ff0a96361c7e49, ASSIGN 2024-11-24T23:13:28,476 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=64743c3cc4326247d7ff0a96361c7e49, ASSIGN; state=OFFLINE, location=ba7f32532130,45393,1732490005487; forceNewPlan=false, retain=false 2024-11-24T23:13:28,477 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5362f4dec21f673244ee873440d1779, ASSIGN 2024-11-24T23:13:28,479 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a5362f4dec21f673244ee873440d1779, ASSIGN; state=OFFLINE, location=ba7f32532130,37739,1732490005584; forceNewPlan=false, retain=false 2024-11-24T23:13:28,481 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:28,482 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN; state=OFFLINE, location=ba7f32532130,36885,1732490005631; forceNewPlan=false, retain=false 2024-11-24T23:13:28,484 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c50f7d0b8194465f687d68a15e4d9d13, ASSIGN 2024-11-24T23:13:28,486 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c50f7d0b8194465f687d68a15e4d9d13, ASSIGN; state=OFFLINE, location=ba7f32532130,45393,1732490005487; forceNewPlan=false, retain=false 2024-11-24T23:13:28,487 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6874a4cd5ca2cb048061687443fd87c9, ASSIGN 2024-11-24T23:13:28,488 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6874a4cd5ca2cb048061687443fd87c9, ASSIGN; state=OFFLINE, location=ba7f32532130,37739,1732490005584; forceNewPlan=false, retain=false 2024-11-24T23:13:28,489 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:28,491 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN; state=OFFLINE, location=ba7f32532130,36885,1732490005631; forceNewPlan=false, retain=false 2024-11-24T23:13:28,492 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7a1389df947c85301aa553a546616498, ASSIGN 2024-11-24T23:13:28,493 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7a1389df947c85301aa553a546616498, ASSIGN; state=OFFLINE, location=ba7f32532130,45393,1732490005487; forceNewPlan=false, retain=false 2024-11-24T23:13:28,494 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a817676ac0dff05fc27e6ec9f8e6c89, ASSIGN 2024-11-24T23:13:28,495 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6a817676ac0dff05fc27e6ec9f8e6c89, ASSIGN; state=OFFLINE, location=ba7f32532130,37739,1732490005584; forceNewPlan=false, retain=false 2024-11-24T23:13:28,497 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:28,498 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN; state=OFFLINE, location=ba7f32532130,36885,1732490005631; forceNewPlan=false, retain=false 2024-11-24T23:13:28,499 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=69fc34e6caf2036ee015474372306228, ASSIGN 2024-11-24T23:13:28,500 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=69fc34e6caf2036ee015474372306228, ASSIGN; state=OFFLINE, location=ba7f32532130,45393,1732490005487; forceNewPlan=false, retain=false 2024-11-24T23:13:28,502 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b32141aa52f6862fa21af187e3ac3b72, ASSIGN 2024-11-24T23:13:28,503 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b32141aa52f6862fa21af187e3ac3b72, ASSIGN; state=OFFLINE, location=ba7f32532130,37739,1732490005584; forceNewPlan=false, retain=false 2024-11-24T23:13:28,505 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:28,506 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN; state=OFFLINE, location=ba7f32532130,36885,1732490005631; forceNewPlan=false, retain=false 2024-11-24T23:13:28,508 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df65ea81e19f0ed6cb718601a302e9ef, ASSIGN 2024-11-24T23:13:28,509 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=df65ea81e19f0ed6cb718601a302e9ef, ASSIGN; state=OFFLINE, location=ba7f32532130,37739,1732490005584; forceNewPlan=false, retain=false 2024-11-24T23:13:28,510 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97ef64c446508710b8b41414b50b97d8, ASSIGN 2024-11-24T23:13:28,512 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=97ef64c446508710b8b41414b50b97d8, ASSIGN; state=OFFLINE, location=ba7f32532130,45393,1732490005487; forceNewPlan=false, retain=false 2024-11-24T23:13:28,513 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:28,514 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN; state=OFFLINE, location=ba7f32532130,36885,1732490005631; forceNewPlan=false, retain=false 2024-11-24T23:13:28,515 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=740a2018f9a8f09b78987d38b376567b, ASSIGN 2024-11-24T23:13:28,516 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=740a2018f9a8f09b78987d38b376567b, ASSIGN; state=OFFLINE, location=ba7f32532130,37739,1732490005584; forceNewPlan=false, retain=false 2024-11-24T23:13:28,518 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79f30edc5579a1fb86c63410b3b6d59a, ASSIGN 2024-11-24T23:13:28,519 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=79f30edc5579a1fb86c63410b3b6d59a, ASSIGN; state=OFFLINE, location=ba7f32532130,45393,1732490005487; forceNewPlan=false, retain=false 2024-11-24T23:13:28,520 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:28,521 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN; state=OFFLINE, location=ba7f32532130,36885,1732490005631; forceNewPlan=false, retain=false 2024-11-24T23:13:28,522 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0ab12bee6921e08663b5265ccf589a9d, ASSIGN 2024-11-24T23:13:28,523 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0ab12bee6921e08663b5265ccf589a9d, ASSIGN; state=OFFLINE, location=ba7f32532130,45393,1732490005487; forceNewPlan=false, retain=false 2024-11-24T23:13:28,525 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8ee88c2c22e3a06388845a14a86f9b2, ASSIGN 2024-11-24T23:13:28,526 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c8ee88c2c22e3a06388845a14a86f9b2, ASSIGN; state=OFFLINE, location=ba7f32532130,37739,1732490005584; forceNewPlan=false, retain=false 2024-11-24T23:13:28,528 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b4c5d46f0872640f4da48af2d12d1131, ASSIGN 2024-11-24T23:13:28,529 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b4c5d46f0872640f4da48af2d12d1131, ASSIGN; state=OFFLINE, location=ba7f32532130,37739,1732490005584; forceNewPlan=false, retain=false 2024-11-24T23:13:28,531 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:28,532 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN; state=OFFLINE, location=ba7f32532130,36885,1732490005631; forceNewPlan=false, retain=false 2024-11-24T23:13:28,533 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=afc068dc7ba93a54eff038a8f06c8542, ASSIGN 2024-11-24T23:13:28,535 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=afc068dc7ba93a54eff038a8f06c8542, ASSIGN; state=OFFLINE, location=ba7f32532130,45393,1732490005487; forceNewPlan=false, retain=false 2024-11-24T23:13:28,625 INFO [ba7f32532130:34129 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-24T23:13:28,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,628 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=64743c3cc4326247d7ff0a96361c7e49, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,629 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,630 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7a1389df947c85301aa553a546616498, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,631 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=69fc34e6caf2036ee015474372306228, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,631 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=97ef64c446508710b8b41414b50b97d8, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,632 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,632 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=0ab12bee6921e08663b5265ccf589a9d, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,633 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=afc068dc7ba93a54eff038a8f06c8542, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,633 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=adb73b3b244f40e15cbf9254ee22925b, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2f0e42305404ba59b5243b2bf8292cf0, ASSIGN because future has completed 2024-11-24T23:13:28,634 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=a5362f4dec21f673244ee873440d1779, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,635 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=6874a4cd5ca2cb048061687443fd87c9, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f0e42305404ba59b5243b2bf8292cf0, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:28,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=64743c3cc4326247d7ff0a96361c7e49, ASSIGN because future has completed 2024-11-24T23:13:28,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c50f7d0b8194465f687d68a15e4d9d13, ASSIGN because future has completed 2024-11-24T23:13:28,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7a1389df947c85301aa553a546616498, ASSIGN because future has completed 2024-11-24T23:13:28,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=69fc34e6caf2036ee015474372306228, ASSIGN because future has completed 2024-11-24T23:13:28,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69fc34e6caf2036ee015474372306228, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:28,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=97ef64c446508710b8b41414b50b97d8, ASSIGN because future has completed 2024-11-24T23:13:28,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=79f30edc5579a1fb86c63410b3b6d59a, ASSIGN because future has completed 2024-11-24T23:13:28,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0ab12bee6921e08663b5265ccf589a9d, ASSIGN because future has completed 2024-11-24T23:13:28,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=afc068dc7ba93a54eff038a8f06c8542, ASSIGN because future has completed 2024-11-24T23:13:28,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure afc068dc7ba93a54eff038a8f06c8542, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:28,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=adb73b3b244f40e15cbf9254ee22925b, ASSIGN because future has completed 2024-11-24T23:13:28,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a5362f4dec21f673244ee873440d1779, ASSIGN because future has completed 2024-11-24T23:13:28,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6874a4cd5ca2cb048061687443fd87c9, ASSIGN because future has completed 2024-11-24T23:13:28,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6874a4cd5ca2cb048061687443fd87c9, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:28,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5362f4dec21f673244ee873440d1779, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:28,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure adb73b3b244f40e15cbf9254ee22925b, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:28,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ab12bee6921e08663b5265ccf589a9d, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:28,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79f30edc5579a1fb86c63410b3b6d59a, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:28,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97ef64c446508710b8b41414b50b97d8, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:28,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a1389df947c85301aa553a546616498, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:28,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure c50f7d0b8194465f687d68a15e4d9d13, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:28,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64743c3cc4326247d7ff0a96361c7e49, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:28,710 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b32141aa52f6862fa21af187e3ac3b72, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=740a2018f9a8f09b78987d38b376567b, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,713 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b4c5d46f0872640f4da48af2d12d1131, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,713 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,714 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6a817676ac0dff05fc27e6ec9f8e6c89, ASSIGN because future has completed 2024-11-24T23:13:28,714 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a817676ac0dff05fc27e6ec9f8e6c89, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:28,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b32141aa52f6862fa21af187e3ac3b72, ASSIGN because future has completed 2024-11-24T23:13:28,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=df65ea81e19f0ed6cb718601a302e9ef, ASSIGN because future has completed 2024-11-24T23:13:28,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=740a2018f9a8f09b78987d38b376567b, ASSIGN because future has completed 2024-11-24T23:13:28,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 740a2018f9a8f09b78987d38b376567b, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:28,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c8ee88c2c22e3a06388845a14a86f9b2, ASSIGN because future has completed 2024-11-24T23:13:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T23:13:28,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b4c5d46f0872640f4da48af2d12d1131, ASSIGN because future has completed 2024-11-24T23:13:28,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN because future has completed 2024-11-24T23:13:28,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN because future has completed 2024-11-24T23:13:28,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,36885,1732490005631}] 2024-11-24T23:13:28,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN because future has completed 2024-11-24T23:13:28,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,36885,1732490005631}] 2024-11-24T23:13:28,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,36885,1732490005631}] 2024-11-24T23:13:28,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4c5d46f0872640f4da48af2d12d1131, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:28,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8ee88c2c22e3a06388845a14a86f9b2, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:28,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure df65ea81e19f0ed6cb718601a302e9ef, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:28,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b32141aa52f6862fa21af187e3ac3b72, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:28,754 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,755 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,756 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=6e72e703f9026a43993a24ead31db485, regionState=OPENING, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,756 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,757 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN because future has completed 2024-11-24T23:13:28,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,36885,1732490005631}] 2024-11-24T23:13:28,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN because future has completed 2024-11-24T23:13:28,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN because future has completed 2024-11-24T23:13:28,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN because future has completed 2024-11-24T23:13:28,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,36885,1732490005631}] 2024-11-24T23:13:28,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN because future has completed 2024-11-24T23:13:28,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,36885,1732490005631}] 2024-11-24T23:13:28,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,36885,1732490005631}] 2024-11-24T23:13:28,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,36885,1732490005631}] 2024-11-24T23:13:28,791 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T23:13:28,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41811, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T23:13:28,799 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:28,799 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 97ef64c446508710b8b41414b50b97d8, NAME => 'test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-24T23:13:28,800 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,800 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,800 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,800 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,803 INFO [StoreOpener-97ef64c446508710b8b41414b50b97d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,805 INFO [StoreOpener-97ef64c446508710b8b41414b50b97d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97ef64c446508710b8b41414b50b97d8 columnFamilyName family 2024-11-24T23:13:28,805 DEBUG [StoreOpener-97ef64c446508710b8b41414b50b97d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,806 INFO [StoreOpener-97ef64c446508710b8b41414b50b97d8-1 {}] regionserver.HStore(327): Store=97ef64c446508710b8b41414b50b97d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,806 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,807 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,808 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,809 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,809 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,811 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,815 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/97ef64c446508710b8b41414b50b97d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,816 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 97ef64c446508710b8b41414b50b97d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60142473, jitterRate=-0.10380731523036957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,816 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:28,818 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 97ef64c446508710b8b41414b50b97d8: Running coprocessor pre-open hook at 1732490008800Writing region info on filesystem at 1732490008800Initializing all the Stores at 1732490008802 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008802Cleaning up temporary data from old regions at 1732490008809 (+7 ms)Running coprocessor post-open hooks at 1732490008816 (+7 ms)Region opened successfully at 1732490008817 (+1 ms) 2024-11-24T23:13:28,820 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8., pid=39, masterSystemTime=1732490008791 2024-11-24T23:13:28,820 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T23:13:28,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34125, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T23:13:28,824 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:28,824 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:28,824 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:28,824 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 79f30edc5579a1fb86c63410b3b6d59a, NAME => 'test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-24T23:13:28,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,825 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,825 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,825 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,825 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97ef64c446508710b8b41414b50b97d8, server=ba7f32532130,45393,1732490005487 because future has completed 2024-11-24T23:13:28,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=14 2024-11-24T23:13:28,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 97ef64c446508710b8b41414b50b97d8, server=ba7f32532130,45393,1732490005487 in 142 msec 2024-11-24T23:13:28,835 INFO [StoreOpener-79f30edc5579a1fb86c63410b3b6d59a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,837 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:28,837 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => adb73b3b244f40e15cbf9254ee22925b, NAME => 'test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-24T23:13:28,838 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,838 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,839 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=97ef64c446508710b8b41414b50b97d8, ASSIGN in 377 msec 2024-11-24T23:13:28,839 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,841 INFO [StoreOpener-79f30edc5579a1fb86c63410b3b6d59a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79f30edc5579a1fb86c63410b3b6d59a columnFamilyName family 2024-11-24T23:13:28,841 DEBUG [StoreOpener-79f30edc5579a1fb86c63410b3b6d59a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,842 INFO [StoreOpener-adb73b3b244f40e15cbf9254ee22925b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,842 INFO [StoreOpener-79f30edc5579a1fb86c63410b3b6d59a-1 {}] regionserver.HStore(327): Store=79f30edc5579a1fb86c63410b3b6d59a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,843 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,844 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,844 INFO [StoreOpener-adb73b3b244f40e15cbf9254ee22925b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adb73b3b244f40e15cbf9254ee22925b columnFamilyName family 2024-11-24T23:13:28,844 DEBUG [StoreOpener-adb73b3b244f40e15cbf9254ee22925b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,845 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,845 INFO [StoreOpener-adb73b3b244f40e15cbf9254ee22925b-1 {}] regionserver.HStore(327): Store=adb73b3b244f40e15cbf9254ee22925b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,845 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,845 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,845 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,846 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,847 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,847 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,847 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,847 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,850 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,850 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/79f30edc5579a1fb86c63410b3b6d59a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,851 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 79f30edc5579a1fb86c63410b3b6d59a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69653197, jitterRate=0.03791351616382599}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,851 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:28,852 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 79f30edc5579a1fb86c63410b3b6d59a: Running coprocessor pre-open hook at 1732490008826Writing region info on filesystem at 1732490008826Initializing all the Stores at 1732490008828 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008828Cleaning up temporary data from old regions at 1732490008845 (+17 ms)Running coprocessor post-open hooks at 1732490008851 (+6 ms)Region opened successfully at 1732490008851 2024-11-24T23:13:28,853 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/adb73b3b244f40e15cbf9254ee22925b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,853 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a., pid=38, masterSystemTime=1732490008791 2024-11-24T23:13:28,854 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened adb73b3b244f40e15cbf9254ee22925b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61937593, jitterRate=-0.07705794274806976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,854 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:28,854 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for adb73b3b244f40e15cbf9254ee22925b: Running coprocessor pre-open hook at 1732490008839Writing region info on filesystem at 1732490008839Initializing all the Stores at 1732490008841 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008841Cleaning up temporary data from old regions at 1732490008847 (+6 ms)Running coprocessor post-open hooks at 1732490008854 (+7 ms)Region opened successfully at 1732490008854 2024-11-24T23:13:28,856 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b., pid=36, masterSystemTime=1732490008819 2024-11-24T23:13:28,856 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:28,856 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:28,857 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:28,857 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 64743c3cc4326247d7ff0a96361c7e49, NAME => 'test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-24T23:13:28,857 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,857 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,858 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,858 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,860 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:28,860 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:28,860 INFO [StoreOpener-64743c3cc4326247d7ff0a96361c7e49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,860 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:28,860 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => a5362f4dec21f673244ee873440d1779, NAME => 'test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-24T23:13:28,861 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,861 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,861 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,861 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79f30edc5579a1fb86c63410b3b6d59a, server=ba7f32532130,45393,1732490005487 because future has completed 2024-11-24T23:13:28,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,864 INFO [StoreOpener-a5362f4dec21f673244ee873440d1779-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,866 INFO [StoreOpener-64743c3cc4326247d7ff0a96361c7e49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64743c3cc4326247d7ff0a96361c7e49 columnFamilyName family 2024-11-24T23:13:28,867 DEBUG [StoreOpener-64743c3cc4326247d7ff0a96361c7e49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure adb73b3b244f40e15cbf9254ee22925b, server=ba7f32532130,37739,1732490005584 because future has completed 2024-11-24T23:13:28,867 INFO [StoreOpener-a5362f4dec21f673244ee873440d1779-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5362f4dec21f673244ee873440d1779 columnFamilyName family 2024-11-24T23:13:28,867 DEBUG [StoreOpener-a5362f4dec21f673244ee873440d1779-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,867 INFO [StoreOpener-64743c3cc4326247d7ff0a96361c7e49-1 {}] regionserver.HStore(327): Store=64743c3cc4326247d7ff0a96361c7e49/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,867 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,868 INFO [StoreOpener-a5362f4dec21f673244ee873440d1779-1 {}] regionserver.HStore(327): Store=a5362f4dec21f673244ee873440d1779/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,868 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,869 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,869 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,869 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,869 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,870 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,870 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,870 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,870 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=11 2024-11-24T23:13:28,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 79f30edc5579a1fb86c63410b3b6d59a, server=ba7f32532130,45393,1732490005487 in 183 msec 2024-11-24T23:13:28,873 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,873 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,876 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,37739,1732490005584, table=test, region=adb73b3b244f40e15cbf9254ee22925b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:28,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79f30edc5579a1fb86c63410b3b6d59a, ASSIGN in 416 msec 2024-11-24T23:13:28,879 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/a5362f4dec21f673244ee873440d1779/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,879 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/64743c3cc4326247d7ff0a96361c7e49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,879 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened a5362f4dec21f673244ee873440d1779; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69656250, jitterRate=0.037959009408950806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,879 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:28,880 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for a5362f4dec21f673244ee873440d1779: Running coprocessor pre-open hook at 1732490008862Writing region info on filesystem at 1732490008862Initializing all the Stores at 1732490008863 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008863Cleaning up temporary data from old regions at 1732490008870 (+7 ms)Running coprocessor post-open hooks at 1732490008880 (+10 ms)Region opened successfully at 1732490008880 2024-11-24T23:13:28,880 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 64743c3cc4326247d7ff0a96361c7e49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75035670, jitterRate=0.11811861395835876}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,880 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:28,881 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 64743c3cc4326247d7ff0a96361c7e49: Running coprocessor pre-open hook at 1732490008858Writing region info on filesystem at 1732490008858Initializing all the Stores at 1732490008859 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008859Cleaning up temporary data from old regions at 1732490008870 (+11 ms)Running coprocessor post-open hooks at 1732490008881 (+11 ms)Region opened successfully at 1732490008881 2024-11-24T23:13:28,881 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779., pid=35, masterSystemTime=1732490008819 2024-11-24T23:13:28,883 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49., pid=42, masterSystemTime=1732490008791 2024-11-24T23:13:28,884 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:28,884 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:28,884 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:28,884 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 4120d4bb2e1399a50983e420a0eeded8, NAME => 'test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-24T23:13:28,885 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:28,885 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 6874a4cd5ca2cb048061687443fd87c9, NAME => 'test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-24T23:13:28,885 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,885 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,885 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,885 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,885 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,885 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,885 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,885 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=30 2024-11-24T23:13:28,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure adb73b3b244f40e15cbf9254ee22925b, server=ba7f32532130,37739,1732490005584 in 203 msec 2024-11-24T23:13:28,886 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:28,886 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:28,886 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:28,887 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a1389df947c85301aa553a546616498, NAME => 'test,qqq,1732490008065.7a1389df947c85301aa553a546616498.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-24T23:13:28,887 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,888 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,qqq,1732490008065.7a1389df947c85301aa553a546616498.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,888 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,888 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,888 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,888 INFO [StoreOpener-6874a4cd5ca2cb048061687443fd87c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,889 INFO [StoreOpener-4120d4bb2e1399a50983e420a0eeded8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,892 INFO [StoreOpener-6874a4cd5ca2cb048061687443fd87c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6874a4cd5ca2cb048061687443fd87c9 columnFamilyName family 2024-11-24T23:13:28,892 INFO [StoreOpener-7a1389df947c85301aa553a546616498-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,892 DEBUG [StoreOpener-6874a4cd5ca2cb048061687443fd87c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64743c3cc4326247d7ff0a96361c7e49, server=ba7f32532130,45393,1732490005487 because future has completed 2024-11-24T23:13:28,893 INFO [StoreOpener-6874a4cd5ca2cb048061687443fd87c9-1 {}] regionserver.HStore(327): Store=6874a4cd5ca2cb048061687443fd87c9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,893 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,894 INFO [StoreOpener-4120d4bb2e1399a50983e420a0eeded8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4120d4bb2e1399a50983e420a0eeded8 columnFamilyName family 2024-11-24T23:13:28,894 DEBUG [StoreOpener-4120d4bb2e1399a50983e420a0eeded8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,895 INFO [StoreOpener-7a1389df947c85301aa553a546616498-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a1389df947c85301aa553a546616498 columnFamilyName family 2024-11-24T23:13:28,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=adb73b3b244f40e15cbf9254ee22925b, ASSIGN in 434 msec 2024-11-24T23:13:28,895 DEBUG [StoreOpener-7a1389df947c85301aa553a546616498-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,895 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,895 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,896 INFO [StoreOpener-4120d4bb2e1399a50983e420a0eeded8-1 {}] regionserver.HStore(327): Store=4120d4bb2e1399a50983e420a0eeded8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,896 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,896 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,896 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,897 INFO [StoreOpener-7a1389df947c85301aa553a546616498-1 {}] regionserver.HStore(327): Store=7a1389df947c85301aa553a546616498/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,897 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,898 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,899 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,899 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,900 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,900 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,900 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,900 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,900 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,900 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=27 2024-11-24T23:13:28,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 64743c3cc4326247d7ff0a96361c7e49, server=ba7f32532130,45393,1732490005487 in 191 msec 2024-11-24T23:13:28,904 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6874a4cd5ca2cb048061687443fd87c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,904 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,904 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,904 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 6874a4cd5ca2cb048061687443fd87c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73290026, jitterRate=0.09210649132728577}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,905 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:28,905 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 6874a4cd5ca2cb048061687443fd87c9: Running coprocessor pre-open hook at 1732490008885Writing region info on filesystem at 1732490008885Initializing all the Stores at 1732490008887 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008887Cleaning up temporary data from old regions at 1732490008896 (+9 ms)Running coprocessor post-open hooks at 1732490008905 (+9 ms)Region opened successfully at 1732490008905 2024-11-24T23:13:28,906 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9., pid=34, masterSystemTime=1732490008819 2024-11-24T23:13:28,908 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/4120d4bb2e1399a50983e420a0eeded8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=64743c3cc4326247d7ff0a96361c7e49, ASSIGN in 446 msec 2024-11-24T23:13:28,909 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 4120d4bb2e1399a50983e420a0eeded8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66747449, jitterRate=-0.005385503172874451}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,909 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:28,910 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 4120d4bb2e1399a50983e420a0eeded8: Running coprocessor pre-open hook at 1732490008885Writing region info on filesystem at 1732490008885Initializing all the Stores at 1732490008887 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008887Cleaning up temporary data from old regions at 1732490008900 (+13 ms)Running coprocessor post-open hooks at 1732490008909 (+9 ms)Region opened successfully at 1732490008910 (+1 ms) 2024-11-24T23:13:28,912 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:28,912 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8., pid=54, masterSystemTime=1732490008877 2024-11-24T23:13:28,912 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7a1389df947c85301aa553a546616498/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,913 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:28,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=a5362f4dec21f673244ee873440d1779, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,913 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:28,913 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => df65ea81e19f0ed6cb718601a302e9ef, NAME => 'test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-24T23:13:28,913 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,913 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,914 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,914 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,915 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 7a1389df947c85301aa553a546616498; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62931021, jitterRate=-0.062254711985588074}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,915 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a1389df947c85301aa553a546616498 2024-11-24T23:13:28,915 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 7a1389df947c85301aa553a546616498: Running coprocessor pre-open hook at 1732490008888Writing region info on filesystem at 1732490008888Initializing all the Stores at 1732490008890 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008890Cleaning up temporary data from old regions at 1732490008900 (+10 ms)Running coprocessor post-open hooks at 1732490008915 (+15 ms)Region opened successfully at 1732490008915 2024-11-24T23:13:28,916 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:28,916 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:28,916 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:28,916 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 1244e0d7b7b5b8b2d792af82faee1da6, NAME => 'test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-24T23:13:28,917 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,917 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,917 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4120d4bb2e1399a50983e420a0eeded8, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,917 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,917 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5362f4dec21f673244ee873440d1779, server=ba7f32532130,37739,1732490005584 because future has completed 2024-11-24T23:13:28,918 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732490008065.7a1389df947c85301aa553a546616498., pid=40, masterSystemTime=1732490008791 2024-11-24T23:13:28,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,36885,1732490005631 because future has completed 2024-11-24T23:13:28,921 INFO [StoreOpener-df65ea81e19f0ed6cb718601a302e9ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,921 INFO [StoreOpener-1244e0d7b7b5b8b2d792af82faee1da6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,923 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:28,923 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:28,923 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:28,924 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => c50f7d0b8194465f687d68a15e4d9d13, NAME => 'test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-24T23:13:28,924 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,924 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,924 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,924 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,925 INFO [StoreOpener-1244e0d7b7b5b8b2d792af82faee1da6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1244e0d7b7b5b8b2d792af82faee1da6 columnFamilyName family 2024-11-24T23:13:28,925 INFO [StoreOpener-df65ea81e19f0ed6cb718601a302e9ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df65ea81e19f0ed6cb718601a302e9ef columnFamilyName family 2024-11-24T23:13:28,925 DEBUG [StoreOpener-df65ea81e19f0ed6cb718601a302e9ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,925 DEBUG [StoreOpener-1244e0d7b7b5b8b2d792af82faee1da6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,926 INFO [StoreOpener-1244e0d7b7b5b8b2d792af82faee1da6-1 {}] regionserver.HStore(327): Store=1244e0d7b7b5b8b2d792af82faee1da6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,926 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,926 INFO [StoreOpener-df65ea81e19f0ed6cb718601a302e9ef-1 {}] regionserver.HStore(327): Store=df65ea81e19f0ed6cb718601a302e9ef/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,926 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=26 2024-11-24T23:13:28,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure a5362f4dec21f673244ee873440d1779, server=ba7f32532130,37739,1732490005584 in 251 msec 2024-11-24T23:13:28,927 INFO [StoreOpener-c50f7d0b8194465f687d68a15e4d9d13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,927 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,928 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,929 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,929 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,930 INFO [StoreOpener-c50f7d0b8194465f687d68a15e4d9d13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c50f7d0b8194465f687d68a15e4d9d13 columnFamilyName family 2024-11-24T23:13:28,930 DEBUG [StoreOpener-c50f7d0b8194465f687d68a15e4d9d13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,930 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,931 INFO [StoreOpener-c50f7d0b8194465f687d68a15e4d9d13-1 {}] regionserver.HStore(327): Store=c50f7d0b8194465f687d68a15e4d9d13/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,931 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,931 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a5362f4dec21f673244ee873440d1779, ASSIGN in 470 msec 2024-11-24T23:13:28,932 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,933 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,933 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,933 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,934 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=7a1389df947c85301aa553a546616498, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:28,934 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,936 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,936 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,937 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a1389df947c85301aa553a546616498, server=ba7f32532130,45393,1732490005487 because future has completed 2024-11-24T23:13:28,938 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,941 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/1244e0d7b7b5b8b2d792af82faee1da6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,941 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/df65ea81e19f0ed6cb718601a302e9ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,942 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened df65ea81e19f0ed6cb718601a302e9ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73086594, jitterRate=0.08907511830329895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,942 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 1244e0d7b7b5b8b2d792af82faee1da6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64099843, jitterRate=-0.04483790695667267}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,942 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:28,942 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:28,942 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for df65ea81e19f0ed6cb718601a302e9ef: Running coprocessor pre-open hook at 1732490008914Writing region info on filesystem at 1732490008914Initializing all the Stores at 1732490008918 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008918Cleaning up temporary data from old regions at 1732490008933 (+15 ms)Running coprocessor post-open hooks at 1732490008942 (+9 ms)Region opened successfully at 1732490008942 2024-11-24T23:13:28,942 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 1244e0d7b7b5b8b2d792af82faee1da6: Running coprocessor pre-open hook at 1732490008917Writing region info on filesystem at 1732490008917Initializing all the Stores at 1732490008919 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008919Cleaning up temporary data from old regions at 1732490008929 (+10 ms)Running coprocessor post-open hooks at 1732490008942 (+13 ms)Region opened successfully at 1732490008942 2024-11-24T23:13:28,944 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef., pid=50, masterSystemTime=1732490008819 2024-11-24T23:13:28,944 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6., pid=46, masterSystemTime=1732490008877 2024-11-24T23:13:28,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=21 2024-11-24T23:13:28,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a1389df947c85301aa553a546616498, server=ba7f32532130,45393,1732490005487 in 244 msec 2024-11-24T23:13:28,947 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:28,947 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:28,948 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:28,949 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a817676ac0dff05fc27e6ec9f8e6c89, NAME => 'test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-24T23:13:28,949 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,949 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,949 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,949 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,950 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/c50f7d0b8194465f687d68a15e4d9d13/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7a1389df947c85301aa553a546616498, ASSIGN in 489 msec 2024-11-24T23:13:28,951 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:28,951 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:28,951 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:28,951 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => eb705a3a04d42fd23798b48207c112de, NAME => 'test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-24T23:13:28,951 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened c50f7d0b8194465f687d68a15e4d9d13; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73707313, jitterRate=0.09832455217838287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,951 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:28,951 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for c50f7d0b8194465f687d68a15e4d9d13: Running coprocessor pre-open hook at 1732490008925Writing region info on filesystem at 1732490008925Initializing all the Stores at 1732490008926 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008927 (+1 ms)Cleaning up temporary data from old regions at 1732490008936 (+9 ms)Running coprocessor post-open hooks at 1732490008951 (+15 ms)Region opened successfully at 1732490008951 2024-11-24T23:13:28,951 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,952 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,952 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,952 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,952 INFO [StoreOpener-6a817676ac0dff05fc27e6ec9f8e6c89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,953 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13., pid=41, masterSystemTime=1732490008791 2024-11-24T23:13:28,953 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,955 INFO [StoreOpener-6a817676ac0dff05fc27e6ec9f8e6c89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a817676ac0dff05fc27e6ec9f8e6c89 columnFamilyName family 2024-11-24T23:13:28,955 DEBUG [StoreOpener-6a817676ac0dff05fc27e6ec9f8e6c89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,956 INFO [StoreOpener-6a817676ac0dff05fc27e6ec9f8e6c89-1 {}] regionserver.HStore(327): Store=6a817676ac0dff05fc27e6ec9f8e6c89/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,956 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,957 INFO [StoreOpener-eb705a3a04d42fd23798b48207c112de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,36885,1732490005631 because future has completed 2024-11-24T23:13:28,957 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:28,957 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:28,957 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:28,957 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => afc068dc7ba93a54eff038a8f06c8542, NAME => 'test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-24T23:13:28,958 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,958 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,958 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,958 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,958 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,959 INFO [StoreOpener-eb705a3a04d42fd23798b48207c112de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb705a3a04d42fd23798b48207c112de columnFamilyName family 2024-11-24T23:13:28,959 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,959 DEBUG [StoreOpener-eb705a3a04d42fd23798b48207c112de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,960 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,961 INFO [StoreOpener-afc068dc7ba93a54eff038a8f06c8542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,960 INFO [StoreOpener-eb705a3a04d42fd23798b48207c112de-1 {}] regionserver.HStore(327): Store=eb705a3a04d42fd23798b48207c112de/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,961 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure df65ea81e19f0ed6cb718601a302e9ef, server=ba7f32532130,37739,1732490005584 because future has completed 2024-11-24T23:13:28,962 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,962 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,962 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,963 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,963 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,963 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,964 INFO [StoreOpener-afc068dc7ba93a54eff038a8f06c8542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afc068dc7ba93a54eff038a8f06c8542 columnFamilyName family 2024-11-24T23:13:28,964 DEBUG [StoreOpener-afc068dc7ba93a54eff038a8f06c8542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,964 INFO [StoreOpener-afc068dc7ba93a54eff038a8f06c8542-1 {}] regionserver.HStore(327): Store=afc068dc7ba93a54eff038a8f06c8542/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,965 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,966 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,966 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,966 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,967 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,967 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,967 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=22 2024-11-24T23:13:28,970 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,971 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/eb705a3a04d42fd23798b48207c112de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,36885,1732490005631 in 234 msec 2024-11-24T23:13:28,971 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened eb705a3a04d42fd23798b48207c112de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68010676, jitterRate=0.013438045978546143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,972 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:28,972 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for eb705a3a04d42fd23798b48207c112de: Running coprocessor pre-open hook at 1732490008952Writing region info on filesystem at 1732490008952Initializing all the Stores at 1732490008954 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008954Cleaning up temporary data from old regions at 1732490008963 (+9 ms)Running coprocessor post-open hooks at 1732490008972 (+9 ms)Region opened successfully at 1732490008972 2024-11-24T23:13:28,973 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de., pid=53, masterSystemTime=1732490008877 2024-11-24T23:13:28,974 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6a817676ac0dff05fc27e6ec9f8e6c89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,974 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/afc068dc7ba93a54eff038a8f06c8542/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:28,975 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened afc068dc7ba93a54eff038a8f06c8542; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59950035, jitterRate=-0.10667486488819122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,975 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:28,976 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for afc068dc7ba93a54eff038a8f06c8542: Running coprocessor pre-open hook at 1732490008958Writing region info on filesystem at 1732490008958Initializing all the Stores at 1732490008959 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008959Cleaning up temporary data from old regions at 1732490008967 (+8 ms)Running coprocessor post-open hooks at 1732490008975 (+8 ms)Region opened successfully at 1732490008976 (+1 ms) 2024-11-24T23:13:28,976 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 6a817676ac0dff05fc27e6ec9f8e6c89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59963610, jitterRate=-0.10647258162498474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:28,976 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:28,976 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 6a817676ac0dff05fc27e6ec9f8e6c89: Running coprocessor pre-open hook at 1732490008949Writing region info on filesystem at 1732490008949Initializing all the Stores at 1732490008951 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008951Cleaning up temporary data from old regions at 1732490008962 (+11 ms)Running coprocessor post-open hooks at 1732490008976 (+14 ms)Region opened successfully at 1732490008976 2024-11-24T23:13:28,978 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542., pid=33, masterSystemTime=1732490008791 2024-11-24T23:13:28,978 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89., pid=43, masterSystemTime=1732490008819 2024-11-24T23:13:28,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN in 514 msec 2024-11-24T23:13:28,979 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:28,979 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:28,980 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:28,980 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 7af30662d67e218e5eed0cf39272545b, NAME => 'test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-24T23:13:28,980 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,980 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,36885,1732490005631, table=test, region=4120d4bb2e1399a50983e420a0eeded8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:28,980 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,980 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,980 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=eb705a3a04d42fd23798b48207c112de, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:28,983 INFO [StoreOpener-7af30662d67e218e5eed0cf39272545b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,984 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:28,984 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:28,984 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:28,984 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ab12bee6921e08663b5265ccf589a9d, NAME => 'test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-24T23:13:28,984 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:28,985 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,985 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:28,985 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:28,986 INFO [StoreOpener-7af30662d67e218e5eed0cf39272545b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7af30662d67e218e5eed0cf39272545b columnFamilyName family 2024-11-24T23:13:28,986 DEBUG [StoreOpener-7af30662d67e218e5eed0cf39272545b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,987 INFO [StoreOpener-7af30662d67e218e5eed0cf39272545b-1 {}] regionserver.HStore(327): Store=7af30662d67e218e5eed0cf39272545b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,36885,1732490005631 because future has completed 2024-11-24T23:13:28,987 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,987 INFO [StoreOpener-0ab12bee6921e08663b5265ccf589a9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:28,987 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:28,988 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:28,988 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,37739,1732490005584, table=test, region=df65ea81e19f0ed6cb718601a302e9ef. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:28,988 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,988 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:28,988 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => b4c5d46f0872640f4da48af2d12d1131, NAME => 'test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-24T23:13:28,988 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:28,989 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:28,989 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:28,989 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:28,989 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:28,991 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,993 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,993 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,993 INFO [StoreOpener-0ab12bee6921e08663b5265ccf589a9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ab12bee6921e08663b5265ccf589a9d columnFamilyName family 2024-11-24T23:13:28,993 DEBUG [StoreOpener-0ab12bee6921e08663b5265ccf589a9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:28,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a817676ac0dff05fc27e6ec9f8e6c89, server=ba7f32532130,37739,1732490005584 because future has completed 2024-11-24T23:13:28,994 INFO [StoreOpener-0ab12bee6921e08663b5265ccf589a9d-1 {}] regionserver.HStore(327): Store=0ab12bee6921e08663b5265ccf589a9d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:28,995 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:28,996 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:28,996 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:28,997 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:28,997 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:28,997 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:29,000 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:29,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=10 2024-11-24T23:13:29,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,36885,1732490005631 in 232 msec 2024-11-24T23:13:29,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN in 544 msec 2024-11-24T23:13:29,008 INFO [StoreOpener-b4c5d46f0872640f4da48af2d12d1131-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:29,010 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/0ab12bee6921e08663b5265ccf589a9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,010 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7af30662d67e218e5eed0cf39272545b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,011 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 0ab12bee6921e08663b5265ccf589a9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66096741, jitterRate=-0.015081807971000671}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,011 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:29,011 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 0ab12bee6921e08663b5265ccf589a9d: Running coprocessor pre-open hook at 1732490008985Writing region info on filesystem at 1732490008985Initializing all the Stores at 1732490008986 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008986Cleaning up temporary data from old regions at 1732490008997 (+11 ms)Running coprocessor post-open hooks at 1732490009011 (+14 ms)Region opened successfully at 1732490009011 2024-11-24T23:13:29,011 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 7af30662d67e218e5eed0cf39272545b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66716485, jitterRate=-0.0058469027280807495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,011 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:29,011 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 7af30662d67e218e5eed0cf39272545b: Running coprocessor pre-open hook at 1732490008981Writing region info on filesystem at 1732490008981Initializing all the Stores at 1732490008983 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008983Cleaning up temporary data from old regions at 1732490008993 (+10 ms)Running coprocessor post-open hooks at 1732490009011 (+18 ms)Region opened successfully at 1732490009011 2024-11-24T23:13:29,012 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d., pid=37, masterSystemTime=1732490008791 2024-11-24T23:13:29,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=20 2024-11-24T23:13:29,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 6a817676ac0dff05fc27e6ec9f8e6c89, server=ba7f32532130,37739,1732490005584 in 291 msec 2024-11-24T23:13:29,014 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b., pid=56, masterSystemTime=1732490008877 2024-11-24T23:13:29,015 INFO [StoreOpener-b4c5d46f0872640f4da48af2d12d1131-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4c5d46f0872640f4da48af2d12d1131 columnFamilyName family 2024-11-24T23:13:29,015 DEBUG [StoreOpener-b4c5d46f0872640f4da48af2d12d1131-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,016 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:29,016 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:29,016 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:29,016 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f0e42305404ba59b5243b2bf8292cf0, NAME => 'test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-24T23:13:29,017 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,017 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:29,017 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,017 INFO [StoreOpener-b4c5d46f0872640f4da48af2d12d1131-1 {}] regionserver.HStore(327): Store=b4c5d46f0872640f4da48af2d12d1131/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,017 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,017 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:29,019 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:29,019 INFO [StoreOpener-2f0e42305404ba59b5243b2bf8292cf0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6a817676ac0dff05fc27e6ec9f8e6c89, ASSIGN in 555 msec 2024-11-24T23:13:29,021 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:29,021 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:29,021 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:29,022 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:29,022 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=7af30662d67e218e5eed0cf39272545b, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:29,022 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 90985da3d485ab013ba37f15b715a5d0, NAME => 'test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-24T23:13:29,022 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,022 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:29,022 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,022 INFO [StoreOpener-2f0e42305404ba59b5243b2bf8292cf0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f0e42305404ba59b5243b2bf8292cf0 columnFamilyName family 2024-11-24T23:13:29,022 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,022 DEBUG [StoreOpener-2f0e42305404ba59b5243b2bf8292cf0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,023 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:29,023 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:29,025 INFO [StoreOpener-90985da3d485ab013ba37f15b715a5d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,025 INFO [StoreOpener-2f0e42305404ba59b5243b2bf8292cf0-1 {}] regionserver.HStore(327): Store=2f0e42305404ba59b5243b2bf8292cf0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,025 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,027 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:29,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,36885,1732490005631 because future has completed 2024-11-24T23:13:29,027 INFO [StoreOpener-90985da3d485ab013ba37f15b715a5d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90985da3d485ab013ba37f15b715a5d0 columnFamilyName family 2024-11-24T23:13:29,027 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,027 DEBUG [StoreOpener-90985da3d485ab013ba37f15b715a5d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,028 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:29,028 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,028 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,029 INFO [StoreOpener-90985da3d485ab013ba37f15b715a5d0-1 {}] regionserver.HStore(327): Store=90985da3d485ab013ba37f15b715a5d0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,029 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,030 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,031 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,031 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,031 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,032 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ab12bee6921e08663b5265ccf589a9d, server=ba7f32532130,45393,1732490005487 because future has completed 2024-11-24T23:13:29,033 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/b4c5d46f0872640f4da48af2d12d1131/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,034 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened b4c5d46f0872640f4da48af2d12d1131; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64822001, jitterRate=-0.03407691419124603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,035 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:29,035 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for b4c5d46f0872640f4da48af2d12d1131: Running coprocessor pre-open hook at 1732490008989Writing region info on filesystem at 1732490008989Initializing all the Stores at 1732490008992 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490008992Cleaning up temporary data from old regions at 1732490009023 (+31 ms)Running coprocessor post-open hooks at 1732490009035 (+12 ms)Region opened successfully at 1732490009035 2024-11-24T23:13:29,036 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,036 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131., pid=48, masterSystemTime=1732490008819 2024-11-24T23:13:29,038 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/2f0e42305404ba59b5243b2bf8292cf0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,039 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 2f0e42305404ba59b5243b2bf8292cf0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72215924, jitterRate=0.07610112428665161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,040 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:29,040 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 2f0e42305404ba59b5243b2bf8292cf0: Running coprocessor pre-open hook at 1732490009017Writing region info on filesystem at 1732490009017Initializing all the Stores at 1732490009019 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490009019Cleaning up temporary data from old regions at 1732490009028 (+9 ms)Running coprocessor post-open hooks at 1732490009040 (+12 ms)Region opened successfully at 1732490009040 2024-11-24T23:13:29,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=16 2024-11-24T23:13:29,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,36885,1732490005631 in 258 msec 2024-11-24T23:13:29,041 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/90985da3d485ab013ba37f15b715a5d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,041 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:29,041 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:29,041 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:29,041 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0., pid=31, masterSystemTime=1732490008791 2024-11-24T23:13:29,042 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 90985da3d485ab013ba37f15b715a5d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67671716, jitterRate=0.008387148380279541}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,042 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:29,042 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => c8ee88c2c22e3a06388845a14a86f9b2, NAME => 'test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-24T23:13:29,042 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 90985da3d485ab013ba37f15b715a5d0: Running coprocessor pre-open hook at 1732490009023Writing region info on filesystem at 1732490009023Initializing all the Stores at 1732490009024 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490009024Cleaning up temporary data from old regions at 1732490009032 (+8 ms)Running coprocessor post-open hooks at 1732490009042 (+10 ms)Region opened successfully at 1732490009042 2024-11-24T23:13:29,042 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,042 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:29,043 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,043 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,044 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0., pid=47, masterSystemTime=1732490008877 2024-11-24T23:13:29,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:29,045 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:29,045 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:29,045 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:29,046 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 69fc34e6caf2036ee015474372306228, NAME => 'test,nnn,1732490008065.69fc34e6caf2036ee015474372306228.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-24T23:13:29,046 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,046 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,nnn,1732490008065.69fc34e6caf2036ee015474372306228.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:29,046 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,046 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,048 INFO [StoreOpener-c8ee88c2c22e3a06388845a14a86f9b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,049 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:29,049 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:29,050 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:29,050 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => f6455d829ffb35ae0e811982e8b77549, NAME => 'test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-24T23:13:29,050 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,050 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:29,051 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,051 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,053 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:29,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4c5d46f0872640f4da48af2d12d1131, server=ba7f32532130,37739,1732490005584 because future has completed 2024-11-24T23:13:29,054 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,45393,1732490005487, table=test, region=0ab12bee6921e08663b5265ccf589a9d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:29,054 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,45393,1732490005487, table=test, region=2f0e42305404ba59b5243b2bf8292cf0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:29,057 INFO [StoreOpener-c8ee88c2c22e3a06388845a14a86f9b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8ee88c2c22e3a06388845a14a86f9b2 columnFamilyName family 2024-11-24T23:13:29,057 DEBUG [StoreOpener-c8ee88c2c22e3a06388845a14a86f9b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f0e42305404ba59b5243b2bf8292cf0, server=ba7f32532130,45393,1732490005487 because future has completed 2024-11-24T23:13:29,058 INFO [StoreOpener-c8ee88c2c22e3a06388845a14a86f9b2-1 {}] regionserver.HStore(327): Store=c8ee88c2c22e3a06388845a14a86f9b2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,059 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,060 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,061 INFO [StoreOpener-69fc34e6caf2036ee015474372306228-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,061 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,062 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,062 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,063 INFO [StoreOpener-69fc34e6caf2036ee015474372306228-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69fc34e6caf2036ee015474372306228 columnFamilyName family 2024-11-24T23:13:29,063 DEBUG [StoreOpener-69fc34e6caf2036ee015474372306228-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,064 INFO [StoreOpener-69fc34e6caf2036ee015474372306228-1 {}] regionserver.HStore(327): Store=69fc34e6caf2036ee015474372306228/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,064 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=7 2024-11-24T23:13:29,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure b4c5d46f0872640f4da48af2d12d1131, server=ba7f32532130,37739,1732490005584 in 321 msec 2024-11-24T23:13:29,066 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,066 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b4c5d46f0872640f4da48af2d12d1131, ASSIGN in 607 msec 2024-11-24T23:13:29,067 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,067 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,069 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,070 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-24T23:13:29,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 2f0e42305404ba59b5243b2bf8292cf0, server=ba7f32532130,45393,1732490005487 in 434 msec 2024-11-24T23:13:29,075 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/69fc34e6caf2036ee015474372306228/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,077 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 69fc34e6caf2036ee015474372306228; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67595102, jitterRate=0.007245510816574097}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,077 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:29,077 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 69fc34e6caf2036ee015474372306228: Running coprocessor pre-open hook at 1732490009046Writing region info on filesystem at 1732490009046Initializing all the Stores at 1732490009055 (+9 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490009055Cleaning up temporary data from old regions at 1732490009067 (+12 ms)Running coprocessor post-open hooks at 1732490009077 (+10 ms)Region opened successfully at 1732490009077 2024-11-24T23:13:29,078 INFO [StoreOpener-f6455d829ffb35ae0e811982e8b77549-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,078 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732490008065.69fc34e6caf2036ee015474372306228., pid=32, masterSystemTime=1732490008791 2024-11-24T23:13:29,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2f0e42305404ba59b5243b2bf8292cf0, ASSIGN in 617 msec 2024-11-24T23:13:29,081 INFO [StoreOpener-f6455d829ffb35ae0e811982e8b77549-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6455d829ffb35ae0e811982e8b77549 columnFamilyName family 2024-11-24T23:13:29,081 DEBUG [StoreOpener-f6455d829ffb35ae0e811982e8b77549-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,082 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=90985da3d485ab013ba37f15b715a5d0, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:29,082 INFO [StoreOpener-f6455d829ffb35ae0e811982e8b77549-1 {}] regionserver.HStore(327): Store=f6455d829ffb35ae0e811982e8b77549/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,082 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,084 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,084 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,085 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,085 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,084 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:29,085 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:29,086 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,36885,1732490005631, table=test, region=4120d4bb2e1399a50983e420a0eeded8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:29,086 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,36885,1732490005631, table=test, region=90985da3d485ab013ba37f15b715a5d0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:29,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,36885,1732490005631 because future has completed 2024-11-24T23:13:29,089 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,093 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/f6455d829ffb35ae0e811982e8b77549/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,093 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/c8ee88c2c22e3a06388845a14a86f9b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,094 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened f6455d829ffb35ae0e811982e8b77549; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58940154, jitterRate=-0.12172326445579529}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,094 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened c8ee88c2c22e3a06388845a14a86f9b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68192677, jitterRate=0.01615007221698761}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,094 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:29,094 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:29,094 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for f6455d829ffb35ae0e811982e8b77549: Running coprocessor pre-open hook at 1732490009051Writing region info on filesystem at 1732490009051Initializing all the Stores at 1732490009056 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490009056Cleaning up temporary data from old regions at 1732490009085 (+29 ms)Running coprocessor post-open hooks at 1732490009094 (+9 ms)Region opened successfully at 1732490009094 2024-11-24T23:13:29,094 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for c8ee88c2c22e3a06388845a14a86f9b2: Running coprocessor pre-open hook at 1732490009043Writing region info on filesystem at 1732490009043Initializing all the Stores at 1732490009046 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490009046Cleaning up temporary data from old regions at 1732490009062 (+16 ms)Running coprocessor post-open hooks at 1732490009094 (+32 ms)Region opened successfully at 1732490009094 2024-11-24T23:13:29,095 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,37739,1732490005584, table=test, region=df65ea81e19f0ed6cb718601a302e9ef. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:29,095 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2., pid=49, masterSystemTime=1732490008819 2024-11-24T23:13:29,095 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549., pid=45, masterSystemTime=1732490008877 2024-11-24T23:13:29,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=28 2024-11-24T23:13:29,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,36885,1732490005631 in 356 msec 2024-11-24T23:13:29,098 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:29,098 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:29,098 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 792edfc226c30220c15d3c2a8c9ffa60, NAME => 'test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-24T23:13:29,099 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f6455d829ffb35ae0e811982e8b77549, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,099 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:29,099 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:29,100 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:29,100 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => b32141aa52f6862fa21af187e3ac3b72, NAME => 'test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-24T23:13:29,100 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,100 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:29,100 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,100 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,36885,1732490005631 because future has completed 2024-11-24T23:13:29,104 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:29,104 INFO [StoreOpener-b32141aa52f6862fa21af187e3ac3b72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,104 INFO [StoreOpener-792edfc226c30220c15d3c2a8c9ffa60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,107 INFO [StoreOpener-792edfc226c30220c15d3c2a8c9ffa60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 792edfc226c30220c15d3c2a8c9ffa60 columnFamilyName family 2024-11-24T23:13:29,107 INFO [StoreOpener-b32141aa52f6862fa21af187e3ac3b72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b32141aa52f6862fa21af187e3ac3b72 columnFamilyName family 2024-11-24T23:13:29,107 DEBUG [StoreOpener-792edfc226c30220c15d3c2a8c9ffa60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,107 DEBUG [StoreOpener-b32141aa52f6862fa21af187e3ac3b72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8ee88c2c22e3a06388845a14a86f9b2, server=ba7f32532130,37739,1732490005584 because future has completed 2024-11-24T23:13:29,108 INFO [StoreOpener-b32141aa52f6862fa21af187e3ac3b72-1 {}] regionserver.HStore(327): Store=b32141aa52f6862fa21af187e3ac3b72/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,108 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,109 INFO [StoreOpener-792edfc226c30220c15d3c2a8c9ffa60-1 {}] regionserver.HStore(327): Store=792edfc226c30220c15d3c2a8c9ffa60/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,109 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,110 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,110 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,110 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,111 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,111 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,111 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,111 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,112 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,113 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,114 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=25 2024-11-24T23:13:29,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,36885,1732490005631 in 385 msec 2024-11-24T23:13:29,117 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/b32141aa52f6862fa21af187e3ac3b72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,118 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened b32141aa52f6862fa21af187e3ac3b72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59418810, jitterRate=-0.11459073424339294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,118 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:29,119 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for b32141aa52f6862fa21af187e3ac3b72: Running coprocessor pre-open hook at 1732490009101Writing region info on filesystem at 1732490009101Initializing all the Stores at 1732490009102 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490009102Cleaning up temporary data from old regions at 1732490009111 (+9 ms)Running coprocessor post-open hooks at 1732490009118 (+7 ms)Region opened successfully at 1732490009118 2024-11-24T23:13:29,119 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/792edfc226c30220c15d3c2a8c9ffa60/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN in 659 msec 2024-11-24T23:13:29,120 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72., pid=51, masterSystemTime=1732490008819 2024-11-24T23:13:29,120 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 792edfc226c30220c15d3c2a8c9ffa60; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67233326, jitterRate=0.001854628324508667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,121 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:29,121 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 792edfc226c30220c15d3c2a8c9ffa60: Running coprocessor pre-open hook at 1732490009099Writing region info on filesystem at 1732490009099Initializing all the Stores at 1732490009101 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490009101Cleaning up temporary data from old regions at 1732490009112 (+11 ms)Running coprocessor post-open hooks at 1732490009121 (+9 ms)Region opened successfully at 1732490009121 2024-11-24T23:13:29,122 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60., pid=52, masterSystemTime=1732490008877 2024-11-24T23:13:29,123 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:29,123 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:29,123 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:29,123 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 740a2018f9a8f09b78987d38b376567b, NAME => 'test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-24T23:13:29,124 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,124 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:29,124 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,124 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,125 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:29,125 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:29,125 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:29,125 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e72e703f9026a43993a24ead31db485, NAME => 'test,iii,1732490008065.6e72e703f9026a43993a24ead31db485.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-24T23:13:29,126 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,126 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,iii,1732490008065.6e72e703f9026a43993a24ead31db485.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:29,126 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,126 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,127 INFO [StoreOpener-740a2018f9a8f09b78987d38b376567b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=8 2024-11-24T23:13:29,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure c8ee88c2c22e3a06388845a14a86f9b2, server=ba7f32532130,37739,1732490005584 in 380 msec 2024-11-24T23:13:29,129 INFO [StoreOpener-740a2018f9a8f09b78987d38b376567b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 740a2018f9a8f09b78987d38b376567b columnFamilyName family 2024-11-24T23:13:29,129 DEBUG [StoreOpener-740a2018f9a8f09b78987d38b376567b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,129 INFO [StoreOpener-6e72e703f9026a43993a24ead31db485-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,129 INFO [StoreOpener-740a2018f9a8f09b78987d38b376567b-1 {}] regionserver.HStore(327): Store=740a2018f9a8f09b78987d38b376567b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,129 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,131 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,132 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,132 INFO [StoreOpener-6e72e703f9026a43993a24ead31db485-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e72e703f9026a43993a24ead31db485 columnFamilyName family 2024-11-24T23:13:29,132 DEBUG [StoreOpener-6e72e703f9026a43993a24ead31db485-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:29,133 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c8ee88c2c22e3a06388845a14a86f9b2, ASSIGN in 672 msec 2024-11-24T23:13:29,133 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,133 INFO [StoreOpener-6e72e703f9026a43993a24ead31db485-1 {}] regionserver.HStore(327): Store=6e72e703f9026a43993a24ead31db485/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:29,133 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,134 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,135 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:29,135 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,136 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,136 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,136 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,36885,1732490005631 because future has completed 2024-11-24T23:13:29,139 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:29,140 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,141 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/740a2018f9a8f09b78987d38b376567b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b32141aa52f6862fa21af187e3ac3b72, server=ba7f32532130,37739,1732490005584 because future has completed 2024-11-24T23:13:29,144 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 740a2018f9a8f09b78987d38b376567b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65400225, jitterRate=-0.025460705161094666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,144 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:29,144 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 740a2018f9a8f09b78987d38b376567b: Running coprocessor pre-open hook at 1732490009124Writing region info on filesystem at 1732490009124Initializing all the Stores at 1732490009126 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490009126Cleaning up temporary data from old regions at 1732490009133 (+7 ms)Running coprocessor post-open hooks at 1732490009144 (+11 ms)Region opened successfully at 1732490009144 2024-11-24T23:13:29,145 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b., pid=44, masterSystemTime=1732490008819 2024-11-24T23:13:29,145 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6e72e703f9026a43993a24ead31db485/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T23:13:29,146 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 6e72e703f9026a43993a24ead31db485; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71969472, jitterRate=0.07242870330810547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:29,146 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:29,147 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 6e72e703f9026a43993a24ead31db485: Running coprocessor pre-open hook at 1732490009126Writing region info on filesystem at 1732490009126Initializing all the Stores at 1732490009129 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490009129Cleaning up temporary data from old regions at 1732490009136 (+7 ms)Running coprocessor post-open hooks at 1732490009146 (+10 ms)Region opened successfully at 1732490009146 2024-11-24T23:13:29,148 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732490008065.6e72e703f9026a43993a24ead31db485., pid=55, masterSystemTime=1732490008877 2024-11-24T23:13:29,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=17 2024-11-24T23:13:29,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure b32141aa52f6862fa21af187e3ac3b72, server=ba7f32532130,37739,1732490005584 in 393 msec 2024-11-24T23:13:29,148 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:29,148 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:29,150 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:29,151 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:29,151 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:29,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 740a2018f9a8f09b78987d38b376567b, server=ba7f32532130,37739,1732490005584 because future has completed 2024-11-24T23:13:29,154 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=6e72e703f9026a43993a24ead31db485, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36885,1732490005631 2024-11-24T23:13:29,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,36885,1732490005631 because future has completed 2024-11-24T23:13:29,161 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,45393,1732490005487, table=test, region=0ab12bee6921e08663b5265ccf589a9d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:29,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=12 2024-11-24T23:13:29,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 740a2018f9a8f09b78987d38b376567b, server=ba7f32532130,37739,1732490005584 in 441 msec 2024-11-24T23:13:29,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=740a2018f9a8f09b78987d38b376567b, ASSIGN in 707 msec 2024-11-24T23:13:29,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=13 2024-11-24T23:13:29,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,36885,1732490005631 in 397 msec 2024-11-24T23:13:29,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN in 714 msec 2024-11-24T23:13:29,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b32141aa52f6862fa21af187e3ac3b72, ASSIGN in 716 msec 2024-11-24T23:13:29,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=19 2024-11-24T23:13:29,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,36885,1732490005631 in 419 msec 2024-11-24T23:13:29,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN in 723 msec 2024-11-24T23:13:29,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN in 727 msec 2024-11-24T23:13:29,188 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=69fc34e6caf2036ee015474372306228, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:29,190 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,36885,1732490005631, table=test, region=4120d4bb2e1399a50983e420a0eeded8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:29,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69fc34e6caf2036ee015474372306228, server=ba7f32532130,45393,1732490005487 because future has completed 2024-11-24T23:13:29,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN in 734 msec 2024-11-24T23:13:29,199 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ba7f32532130,37739,1732490005584, table=test, region=df65ea81e19f0ed6cb718601a302e9ef. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T23:13:29,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=18 2024-11-24T23:13:29,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 69fc34e6caf2036ee015474372306228, server=ba7f32532130,45393,1732490005487 in 549 msec 2024-11-24T23:13:29,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=69fc34e6caf2036ee015474372306228, ASSIGN in 743 msec 2024-11-24T23:13:29,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=9 2024-11-24T23:13:29,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ab12bee6921e08663b5265ccf589a9d, server=ba7f32532130,45393,1732490005487 in 525 msec 2024-11-24T23:13:29,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0ab12bee6921e08663b5265ccf589a9d, ASSIGN in 751 msec 2024-11-24T23:13:29,212 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:29,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure afc068dc7ba93a54eff038a8f06c8542, server=ba7f32532130,45393,1732490005487 because future has completed 2024-11-24T23:13:29,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=15 2024-11-24T23:13:29,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure df65ea81e19f0ed6cb718601a302e9ef, server=ba7f32532130,37739,1732490005584 in 468 msec 2024-11-24T23:13:29,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=df65ea81e19f0ed6cb718601a302e9ef, ASSIGN in 762 msec 2024-11-24T23:13:29,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=5 2024-11-24T23:13:29,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure afc068dc7ba93a54eff038a8f06c8542, server=ba7f32532130,45393,1732490005487 in 564 msec 2024-11-24T23:13:29,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=afc068dc7ba93a54eff038a8f06c8542, ASSIGN in 768 msec 2024-11-24T23:13:29,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T23:13:29,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure c50f7d0b8194465f687d68a15e4d9d13, server=ba7f32532130,45393,1732490005487 because future has completed 2024-11-24T23:13:29,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=6 2024-11-24T23:13:29,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,36885,1732490005631 in 465 msec 2024-11-24T23:13:29,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN in 779 msec 2024-11-24T23:13:29,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=24 2024-11-24T23:13:29,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure c50f7d0b8194465f687d68a15e4d9d13, server=ba7f32532130,45393,1732490005487 in 539 msec 2024-11-24T23:13:29,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c50f7d0b8194465f687d68a15e4d9d13, ASSIGN in 785 msec 2024-11-24T23:13:29,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:29,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6874a4cd5ca2cb048061687443fd87c9, server=ba7f32532130,37739,1732490005584 because future has completed 2024-11-24T23:13:29,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=23 2024-11-24T23:13:29,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 6874a4cd5ca2cb048061687443fd87c9, server=ba7f32532130,37739,1732490005584 in 585 msec 2024-11-24T23:13:29,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=4 2024-11-24T23:13:29,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6874a4cd5ca2cb048061687443fd87c9, ASSIGN in 797 msec 2024-11-24T23:13:29,260 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T23:13:29,260 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732490009260"}]},"ts":"1732490009260"} 2024-11-24T23:13:29,263 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-24T23:13:29,265 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T23:13:29,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.1920 sec 2024-11-24T23:13:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T23:13:30,242 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-24T23:13:30,242 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-24T23:13:30,245 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T23:13:30,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-24T23:13:30,266 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T23:13:30,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-24T23:13:30,267 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T23:13:30,272 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-24T23:13:30,295 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-24T23:13:30,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T23:13:30,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T23:13:30,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T23:13:30,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T23:13:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4120d4bb2e1399a50983e420a0eeded8 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eb705a3a04d42fd23798b48207c112de changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6e72e703f9026a43993a24ead31db485 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7af30662d67e218e5eed0cf39272545b changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 792edfc226c30220c15d3c2a8c9ffa60 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1244e0d7b7b5b8b2d792af82faee1da6 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f6455d829ffb35ae0e811982e8b77549 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 90985da3d485ab013ba37f15b715a5d0 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b4c5d46f0872640f4da48af2d12d1131 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c8ee88c2c22e3a06388845a14a86f9b2 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 740a2018f9a8f09b78987d38b376567b changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region df65ea81e19f0ed6cb718601a302e9ef changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b32141aa52f6862fa21af187e3ac3b72 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6a817676ac0dff05fc27e6ec9f8e6c89 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6874a4cd5ca2cb048061687443fd87c9 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a5362f4dec21f673244ee873440d1779 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region adb73b3b244f40e15cbf9254ee22925b changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region afc068dc7ba93a54eff038a8f06c8542 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0ab12bee6921e08663b5265ccf589a9d changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 79f30edc5579a1fb86c63410b3b6d59a changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 97ef64c446508710b8b41414b50b97d8 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 69fc34e6caf2036ee015474372306228 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7a1389df947c85301aa553a546616498 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c50f7d0b8194465f687d68a15e4d9d13 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 64743c3cc4326247d7ff0a96361c7e49 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2f0e42305404ba59b5243b2bf8292cf0 changed from -1.0 to 0.0, refreshing cache 2024-11-24T23:13:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-24T23:13:30,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-24T23:13:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-24T23:13:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(204): Hosts are {ba7f32532130=0} racks are {/default-rack=0} 2024-11-24T23:13:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-24T23:13:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-24T23:13:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-24T23:13:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T23:13:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T23:13:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T23:13:30,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T23:13:30,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T23:13:30,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T23:13:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-24T23:13:30,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-24T23:13:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-24T23:13:30,376 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T23:13:30,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-24T23:13:30,399 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=386, OpenFileDescriptor=645, MaxFileDescriptor=1048576, SystemLoadAverage=295, ProcessCount=11, AvailableMemoryMB=7365 2024-11-24T23:13:30,403 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-24T23:13:30,403 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-24T23:13:30,403 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing ba7f32532130,36885,1732490005631 2024-11-24T23:13:30,404 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server ba7f32532130,36885,1732490005631: Simulated kill ***** 2024-11-24T23:13:30,404 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T23:13:30,406 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T23:13:30,408 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T23:13:30,408 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T23:13:30,409 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T23:13:30,409 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 310045344 }, "NonHeapMemoryUsage": { "committed": 151846912, "init": 7667712, "max": -1, "used": 149493792 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ba7f32532130" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ba7f32532130" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "ba7f32532130" } ] } 2024-11-24T23:13:30,411 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34129 {}] master.MasterRpcServices(700): ba7f32532130,36885,1732490005631 reported a fatal error: ***** ABORTING region server ba7f32532130,36885,1732490005631: Simulated kill ***** 2024-11-24T23:13:30,413 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,36885,1732490005631' ***** 2024-11-24T23:13:30,413 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-24T23:13:30,413 INFO [RS:2;ba7f32532130:36885 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T23:13:30,414 INFO [RS:2;ba7f32532130:36885 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T23:13:30,414 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T23:13:30,414 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T23:13:30,414 INFO [RS:2;ba7f32532130:36885 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T23:13:30,414 INFO [RS:2;ba7f32532130:36885 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T23:13:30,414 INFO [RS:2;ba7f32532130:36885 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ba7f32532130:36885. 2024-11-24T23:13:30,414 DEBUG [RS:2;ba7f32532130:36885 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T23:13:30,418 DEBUG [RS:2;ba7f32532130:36885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:30,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741836_1012 (size=32114) 2024-11-24T23:13:30,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741836_1012 (size=32114) 2024-11-24T23:13:30,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741836_1012 (size=32114) 2024-11-24T23:13:30,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741833_1009 (size=2237) 2024-11-24T23:13:30,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741833_1009 (size=2237) 2024-11-24T23:13:30,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741833_1009 (size=2237) 2024-11-24T23:13:30,430 DEBUG [RS:2;ba7f32532130:36885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:30,431 INFO [RS:2;ba7f32532130:36885 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T23:13:30,431 INFO [RS:2;ba7f32532130:36885 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36885 2024-11-24T23:13:30,479 INFO [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T23:13:30,552 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,36885,1732490005631 2024-11-24T23:13:30,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T23:13:30,552 INFO [RS:2;ba7f32532130:36885 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T23:13:30,567 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,36885,1732490005631] 2024-11-24T23:13:30,577 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,36885,1732490005631 already deleted, retry=false 2024-11-24T23:13:30,578 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ba7f32532130,36885,1732490005631 on ba7f32532130,34129,1732490004717 2024-11-24T23:13:30,586 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:30,590 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for ba7f32532130,36885,1732490005631 (carryingMeta=true) ba7f32532130,36885,1732490005631/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1bec0139[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T23:13:30,590 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:30,592 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true, isMeta: true 2024-11-24T23:13:30,594 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting 2024-11-24T23:13:30,596 INFO [PEWorker-1 {}] master.SplitWALManager(105): ba7f32532130,36885,1732490005631 WAL count=1, meta=true 2024-11-24T23:13:30,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta}] 2024-11-24T23:13:30,606 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:30,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:30,616 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T23:13:30,667 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T23:13:30,667 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36885-0x1016f1f81790003, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T23:13:30,668 INFO [RS:2;ba7f32532130:36885 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T23:13:30,668 INFO [RS:2;ba7f32532130:36885 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,36885,1732490005631; zookeeper connection closed. 2024-11-24T23:13:30,669 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3ea9e192 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3ea9e192 2024-11-24T23:13:30,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-24T23:13:30,797 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, size=31.4 K (32114bytes) 2024-11-24T23:13:30,798 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T23:13:30,798 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T23:13:30,798 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta 2024-11-24T23:13:30,803 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta after 3ms 2024-11-24T23:13:30,810 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta took 14ms 2024-11-24T23:13:30,818 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-24T23:13:30,825 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta.temp 2024-11-24T23:13:30,828 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta so closing down 2024-11-24T23:13:30,829 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T23:13:30,830 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta.temp 2024-11-24T23:13:30,834 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T23:13:30,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741864_1040 (size=31873) 2024-11-24T23:13:30,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741864_1040 (size=31873) 2024-11-24T23:13:30,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741864_1040 (size=31873) 2024-11-24T23:13:30,846 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-24T23:13:30,848 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta.temp to hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-24T23:13:30,851 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-24T23:13:30,852 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, journal: Splitting hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, size=31.4 K (32114bytes) at 1732490010797Finishing writing output for hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta so closing down at 1732490010829 (+32 ms)Creating recovered edits writer path=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta.temp at 1732490010830 (+1 ms)3 split writer threads finished at 1732490010834 (+4 ms)Closed recovered edits writer path=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1732490010846 (+12 ms)Rename recovered edits hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta.temp to hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732490010848 (+2 ms)Processed 58 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1732490010851 (+3 ms) 2024-11-24T23:13:30,852 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta 2024-11-24T23:13:30,853 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-24T23:13:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-24T23:13:30,861 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta to hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs 2024-11-24T23:13:30,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-24T23:13:30,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 in 253 msec 2024-11-24T23:13:30,865 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:30,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-24T23:13:30,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 in 268 msec 2024-11-24T23:13:30,871 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting dir is empty, no logs to split. 2024-11-24T23:13:30,871 INFO [PEWorker-1 {}] master.SplitWALManager(105): ba7f32532130,36885,1732490005631 WAL count=0, meta=true 2024-11-24T23:13:30,871 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ba7f32532130,36885,1732490005631 WAL splitting is done? wals=0, meta=true 2024-11-24T23:13:30,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T23:13:30,874 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:30,875 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T23:13:31,026 DEBUG [ba7f32532130:34129 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T23:13:31,026 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(204): Hosts are {ba7f32532130=0} racks are {/default-rack=0} 2024-11-24T23:13:31,026 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T23:13:31,026 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T23:13:31,026 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T23:13:31,027 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T23:13:31,027 INFO [ba7f32532130:34129 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T23:13:31,027 INFO [ba7f32532130:34129 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T23:13:31,027 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T23:13:31,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:31,028 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPENING 2024-11-24T23:13:31,076 INFO [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T23:13:31,076 INFO [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T23:13:31,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:31,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:31,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:31,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:31,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:31,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:31,099 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:31,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:31,261 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T23:13:31,262 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T23:13:31,262 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T23:13:31,265 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C37739%2C1732490005584.meta, suffix=.meta, logDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,37739,1732490005584, archiveDir=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs, maxLogs=32 2024-11-24T23:13:31,280 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,37739,1732490005584/ba7f32532130%2C37739%2C1732490005584.meta.1732490011266.meta, exclude list is [], retry=0 2024-11-24T23:13:31,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40001,DS-be315e5e-9a06-443e-884f-6714183c1b21,DISK] 2024-11-24T23:13:31,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39131,DS-a0b0d19b-e93b-4f83-a9ec-e0dbbf30d950,DISK] 2024-11-24T23:13:31,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44853,DS-2b455eb7-85ba-41ff-ac8b-19e614915e5f,DISK] 2024-11-24T23:13:31,285 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,37739,1732490005584/ba7f32532130%2C37739%2C1732490005584.meta.1732490011266.meta 2024-11-24T23:13:31,286 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:41507:41507),(127.0.0.1/127.0.0.1:46315:46315)] 2024-11-24T23:13:31,286 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T23:13:31,286 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T23:13:31,286 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T23:13:31,287 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T23:13:31,287 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T23:13:31,287 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:31,287 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T23:13:31,287 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T23:13:31,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T23:13:31,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T23:13:31,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:31,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:31,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T23:13:31,292 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T23:13:31,292 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:31,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:31,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T23:13:31,293 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T23:13:31,293 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:31,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:31,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T23:13:31,295 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T23:13:31,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:31,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T23:13:31,295 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T23:13:31,297 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740 2024-11-24T23:13:31,299 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740 2024-11-24T23:13:31,300 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-24T23:13:31,317 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-24T23:13:31,320 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-24T23:13:31,387 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/info/c290f301efe64c49b8d3bb122f94d96b is 138, key is test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8./info:regioninfo/1732490008917/Put/seqid=0 2024-11-24T23:13:31,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741866_1042 (size=34924) 2024-11-24T23:13:31,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741866_1042 (size=34924) 2024-11-24T23:13:31,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741866_1042 (size=34924) 2024-11-24T23:13:31,404 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/info/c290f301efe64c49b8d3bb122f94d96b 2024-11-24T23:13:31,484 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/ns/6c06063ae3a2427288104042d29e54f9 is 43, key is default/ns:d/1732490007787/Put/seqid=0 2024-11-24T23:13:31,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741867_1043 (size=5153) 2024-11-24T23:13:31,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741867_1043 (size=5153) 2024-11-24T23:13:31,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741867_1043 (size=5153) 2024-11-24T23:13:31,493 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/ns/6c06063ae3a2427288104042d29e54f9 2024-11-24T23:13:31,518 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/table/e8ab7ae9cb874bb2991d0dc51a217807 is 40, key is test/table:state/1732490009260/Put/seqid=0 2024-11-24T23:13:31,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741868_1044 (size=5165) 2024-11-24T23:13:31,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741868_1044 (size=5165) 2024-11-24T23:13:31,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741868_1044 (size=5165) 2024-11-24T23:13:31,526 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/table/e8ab7ae9cb874bb2991d0dc51a217807 2024-11-24T23:13:31,534 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/info/c290f301efe64c49b8d3bb122f94d96b as hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/info/c290f301efe64c49b8d3bb122f94d96b 2024-11-24T23:13:31,542 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/info/c290f301efe64c49b8d3bb122f94d96b, entries=260, sequenceid=60, filesize=34.1 K 2024-11-24T23:13:31,545 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/ns/6c06063ae3a2427288104042d29e54f9 as hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/ns/6c06063ae3a2427288104042d29e54f9 2024-11-24T23:13:31,554 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/ns/6c06063ae3a2427288104042d29e54f9, entries=2, sequenceid=60, filesize=5.0 K 2024-11-24T23:13:31,555 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/table/e8ab7ae9cb874bb2991d0dc51a217807 as hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/table/e8ab7ae9cb874bb2991d0dc51a217807 2024-11-24T23:13:31,565 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/table/e8ab7ae9cb874bb2991d0dc51a217807, entries=2, sequenceid=60, filesize=5.0 K 2024-11-24T23:13:31,570 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 248ms, sequenceid=60, compaction requested=false; wal=null 2024-11-24T23:13:31,571 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T23:13:31,574 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-24T23:13:31,576 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T23:13:31,576 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T23:13:31,576 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T23:13:31,578 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T23:13:31,581 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-24T23:13:31,582 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73933952, jitterRate=0.10170173645019531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T23:13:31,582 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T23:13:31,582 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732490011287Writing region info on filesystem at 1732490011287Initializing all the Stores at 1732490011288 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490011288Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490011289 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490011289Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732490011289Obtaining lock to block concurrent updates at 1732490011320 (+31 ms)Preparing flush snapshotting stores in 1588230740 at 1732490011320Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732490011323 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732490011323Flushing 1588230740/info: creating writer at 1732490011324 (+1 ms)Flushing 1588230740/info: appending metadata at 1732490011374 (+50 ms)Flushing 1588230740/info: closing flushed file at 1732490011377 (+3 ms)Flushing 1588230740/ns: creating writer at 1732490011462 (+85 ms)Flushing 1588230740/ns: appending metadata at 1732490011482 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732490011483 (+1 ms)Flushing 1588230740/table: creating writer at 1732490011502 (+19 ms)Flushing 1588230740/table: appending metadata at 1732490011517 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732490011517Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f705b4: reopening flushed file at 1732490011533 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4658c52b: reopening flushed file at 1732490011543 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33443797: reopening flushed file at 1732490011554 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 248ms, sequenceid=60, compaction requested=false; wal=null at 1732490011570 (+16 ms)Cleaning up temporary data from old regions at 1732490011576 (+6 ms)Running coprocessor post-open hooks at 1732490011582 (+6 ms)Region opened successfully at 1732490011582 2024-11-24T23:13:31,584 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1732490011253 2024-11-24T23:13:31,587 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T23:13:31,587 INFO [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T23:13:31,588 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:31,589 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:31,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:31,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:31,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:31,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:31,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:31,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:31,636 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:31,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-24T23:13:31,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 in 537 msec 2024-11-24T23:13:31,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-24T23:13:31,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 766 msec 2024-11-24T23:13:31,643 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): ba7f32532130,36885,1732490005631 had 9 regions 2024-11-24T23:13:31,644 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true, isMeta: false 2024-11-24T23:13:31,646 INFO [PEWorker-1 {}] master.SplitWALManager(105): ba7f32532130,36885,1732490005631 WAL count=1, meta=false 2024-11-24T23:13:31,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246}] 2024-11-24T23:13:31,649 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:31,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:31,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45393 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-24T23:13:31,842 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246, size=2.2 K (2237bytes) 2024-11-24T23:13:31,842 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246 2024-11-24T23:13:31,843 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246 after 1ms 2024-11-24T23:13:31,850 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246 took 8ms 2024-11-24T23:13:31,858 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4120d4bb2e1399a50983e420a0eeded8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T23:13:31,861 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 1244e0d7b7b5b8b2d792af82faee1da6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T23:13:31,864 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for eb705a3a04d42fd23798b48207c112de: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T23:13:31,869 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7af30662d67e218e5eed0cf39272545b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T23:13:31,871 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 90985da3d485ab013ba37f15b715a5d0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T23:13:31,873 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f6455d829ffb35ae0e811982e8b77549: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T23:13:31,875 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 792edfc226c30220c15d3c2a8c9ffa60: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T23:13:31,878 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 6e72e703f9026a43993a24ead31db485: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T23:13:31,879 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246 so closing down 2024-11-24T23:13:31,879 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T23:13:31,879 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T23:13:31,879 INFO [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 25 ms; skipped=8; WAL=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-24T23:13:31,879 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246, journal: Splitting hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246, size=2.2 K (2237bytes) at 1732490011842Finishing writing output for hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246 so closing down at 1732490011879 (+37 ms)3 split writer threads finished at 1732490011879Processed 8 edits across 0 Regions in 25 ms; skipped=8; WAL=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732490011879 2024-11-24T23:13:31,879 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246 2024-11-24T23:13:31,879 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-24T23:13:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34129 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-24T23:13:31,886 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting/ba7f32532130%2C36885%2C1732490005631.1732490007246 to hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs 2024-11-24T23:13:31,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-24T23:13:31,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 in 236 msec 2024-11-24T23:13:31,891 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:31,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-24T23:13:31,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 in 245 msec 2024-11-24T23:13:31,902 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/WALs/ba7f32532130,36885,1732490005631-splitting dir is empty, no logs to split. 2024-11-24T23:13:31,902 INFO [PEWorker-1 {}] master.SplitWALManager(105): ba7f32532130,36885,1732490005631 WAL count=0, meta=false 2024-11-24T23:13:31,902 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ba7f32532130,36885,1732490005631 WAL splitting is done? wals=0, meta=false 2024-11-24T23:13:31,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN}] 2024-11-24T23:13:31,908 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:31,911 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T23:13:31,913 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:31,915 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T23:13:31,917 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:31,919 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T23:13:31,920 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:31,923 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T23:13:31,925 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:31,929 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T23:13:31,933 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:31,940 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T23:13:31,942 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:31,949 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T23:13:31,953 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:31,954 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T23:13:32,061 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(204): Hosts are {ba7f32532130=0} racks are {/default-rack=0} 2024-11-24T23:13:32,062 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T23:13:32,062 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T23:13:32,062 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T23:13:32,062 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T23:13:32,062 INFO [ba7f32532130:34129 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T23:13:32,062 INFO [ba7f32532130:34129 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T23:13:32,062 DEBUG [ba7f32532130:34129 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T23:13:32,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,063 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,063 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=6e72e703f9026a43993a24ead31db485, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,063 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ba7f32532130:36885 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T23:13:32,064 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, regionLocation=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,064 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,065 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1, error=java.net.ConnectException: Call to address=ba7f32532130:36885 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,065 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,066 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, regionLocation=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T23:13:32,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 from cache 2024-11-24T23:13:32,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address ba7f32532130:36885 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,068 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ba7f32532130:36885 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T23:13:32,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ba7f32532130:36885 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ba7f32532130:36885 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ba7f32532130:36885 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ba7f32532130:36885 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ba7f32532130:36885 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ba7f32532130:36885 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36885,1732490005631, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=ba7f32532130:36885 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address ba7f32532130:36885 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ba7f32532130/172.17.0.3:36885 2024-11-24T23:13:32,170 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T23:13:32,170 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,37739,1732490005584, seqNum=-1] 2024-11-24T23:13:32,170 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T23:13:32,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53105, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T23:13:32,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN because future has completed 2024-11-24T23:13:32,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:32,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN because future has completed 2024-11-24T23:13:32,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN because future has completed 2024-11-24T23:13:32,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN because future has completed 2024-11-24T23:13:32,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:32,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN because future has completed 2024-11-24T23:13:32,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN because future has completed 2024-11-24T23:13:32,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN because future has completed 2024-11-24T23:13:32,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:32,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN because future has completed 2024-11-24T23:13:32,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:32,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584}] 2024-11-24T23:13:32,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:32,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:32,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487}] 2024-11-24T23:13:32,334 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:32,335 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 792edfc226c30220c15d3c2a8c9ffa60, NAME => 'test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-24T23:13:32,335 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,336 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:32,336 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,336 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,339 INFO [StoreOpener-792edfc226c30220c15d3c2a8c9ffa60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,341 INFO [StoreOpener-792edfc226c30220c15d3c2a8c9ffa60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 792edfc226c30220c15d3c2a8c9ffa60 columnFamilyName family 2024-11-24T23:13:32,341 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:32,341 DEBUG [StoreOpener-792edfc226c30220c15d3c2a8c9ffa60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:32,342 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 90985da3d485ab013ba37f15b715a5d0, NAME => 'test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-24T23:13:32,342 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,342 INFO [StoreOpener-792edfc226c30220c15d3c2a8c9ffa60-1 {}] regionserver.HStore(327): Store=792edfc226c30220c15d3c2a8c9ffa60/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:32,342 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:32,342 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,342 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,342 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,343 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,344 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,345 INFO [StoreOpener-90985da3d485ab013ba37f15b715a5d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,345 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,345 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,346 INFO [StoreOpener-90985da3d485ab013ba37f15b715a5d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90985da3d485ab013ba37f15b715a5d0 columnFamilyName family 2024-11-24T23:13:32,346 DEBUG [StoreOpener-90985da3d485ab013ba37f15b715a5d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:32,346 INFO [StoreOpener-90985da3d485ab013ba37f15b715a5d0-1 {}] regionserver.HStore(327): Store=90985da3d485ab013ba37f15b715a5d0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:32,346 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,347 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,347 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,348 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 792edfc226c30220c15d3c2a8c9ffa60; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63533486, jitterRate=-0.053277283906936646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:32,348 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:32,348 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 792edfc226c30220c15d3c2a8c9ffa60: Running coprocessor pre-open hook at 1732490012336Writing region info on filesystem at 1732490012336Initializing all the Stores at 1732490012338 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490012338Cleaning up temporary data from old regions at 1732490012345 (+7 ms)Running coprocessor post-open hooks at 1732490012348 (+3 ms)Region opened successfully at 1732490012348 2024-11-24T23:13:32,349 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,349 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,349 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,350 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60., pid=79, masterSystemTime=1732490012329 2024-11-24T23:13:32,352 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,352 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:32,353 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:32,353 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:32,353 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => f6455d829ffb35ae0e811982e8b77549, NAME => 'test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-24T23:13:32,353 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,354 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:32,354 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 90985da3d485ab013ba37f15b715a5d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74408685, jitterRate=0.10877580940723419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:32,354 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,354 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:32,354 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,354 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 90985da3d485ab013ba37f15b715a5d0: Running coprocessor pre-open hook at 1732490012343Writing region info on filesystem at 1732490012343Initializing all the Stores at 1732490012344 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490012344Cleaning up temporary data from old regions at 1732490012349 (+5 ms)Running coprocessor post-open hooks at 1732490012354 (+5 ms)Region opened successfully at 1732490012354 2024-11-24T23:13:32,355 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0., pid=73, masterSystemTime=1732490012335 2024-11-24T23:13:32,355 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-24T23:13:32,355 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,355 INFO [StoreOpener-f6455d829ffb35ae0e811982e8b77549-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,355 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,357 INFO [StoreOpener-f6455d829ffb35ae0e811982e8b77549-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6455d829ffb35ae0e811982e8b77549 columnFamilyName family 2024-11-24T23:13:32,357 DEBUG [StoreOpener-f6455d829ffb35ae0e811982e8b77549-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:32,358 INFO [StoreOpener-f6455d829ffb35ae0e811982e8b77549-1 {}] regionserver.HStore(327): Store=f6455d829ffb35ae0e811982e8b77549/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:32,358 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:32,358 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,358 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:32,358 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:32,358 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 1244e0d7b7b5b8b2d792af82faee1da6, NAME => 'test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-24T23:13:32,359 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,359 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,359 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:32,359 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,359 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,360 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,360 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,360 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,360 INFO [StoreOpener-1244e0d7b7b5b8b2d792af82faee1da6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,362 INFO [StoreOpener-1244e0d7b7b5b8b2d792af82faee1da6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1244e0d7b7b5b8b2d792af82faee1da6 columnFamilyName family 2024-11-24T23:13:32,362 DEBUG [StoreOpener-1244e0d7b7b5b8b2d792af82faee1da6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:32,362 INFO [StoreOpener-1244e0d7b7b5b8b2d792af82faee1da6-1 {}] regionserver.HStore(327): Store=1244e0d7b7b5b8b2d792af82faee1da6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:32,363 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,363 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,364 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,364 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened f6455d829ffb35ae0e811982e8b77549; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74853199, jitterRate=0.11539958417415619}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:32,364 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:32,365 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for f6455d829ffb35ae0e811982e8b77549: Running coprocessor pre-open hook at 1732490012354Writing region info on filesystem at 1732490012354Initializing all the Stores at 1732490012355 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490012355Cleaning up temporary data from old regions at 1732490012360 (+5 ms)Running coprocessor post-open hooks at 1732490012365 (+5 ms)Region opened successfully at 1732490012365 2024-11-24T23:13:32,366 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549., pid=72, masterSystemTime=1732490012329 2024-11-24T23:13:32,366 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,366 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-24T23:13:32,368 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,368 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,368 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,369 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,369 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:32,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,369 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:32,369 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:32,369 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 4120d4bb2e1399a50983e420a0eeded8, NAME => 'test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-24T23:13:32,370 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,370 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,370 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:32,371 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,371 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,371 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,372 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 1244e0d7b7b5b8b2d792af82faee1da6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68648910, jitterRate=0.022948473691940308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:32,372 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:32,372 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 1244e0d7b7b5b8b2d792af82faee1da6: Running coprocessor pre-open hook at 1732490012359Writing region info on filesystem at 1732490012359Initializing all the Stores at 1732490012360 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490012360Cleaning up temporary data from old regions at 1732490012368 (+8 ms)Running coprocessor post-open hooks at 1732490012372 (+4 ms)Region opened successfully at 1732490012372 2024-11-24T23:13:32,372 INFO [StoreOpener-4120d4bb2e1399a50983e420a0eeded8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,373 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6., pid=74, masterSystemTime=1732490012335 2024-11-24T23:13:32,374 INFO [StoreOpener-4120d4bb2e1399a50983e420a0eeded8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4120d4bb2e1399a50983e420a0eeded8 columnFamilyName family 2024-11-24T23:13:32,374 DEBUG [StoreOpener-4120d4bb2e1399a50983e420a0eeded8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:32,375 INFO [StoreOpener-4120d4bb2e1399a50983e420a0eeded8-1 {}] regionserver.HStore(327): Store=4120d4bb2e1399a50983e420a0eeded8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:32,375 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,376 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,376 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:32,376 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:32,377 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:32,377 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 7af30662d67e218e5eed0cf39272545b, NAME => 'test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-24T23:13:32,377 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,377 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,377 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:32,378 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,378 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,378 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,378 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,379 INFO [StoreOpener-7af30662d67e218e5eed0cf39272545b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,381 INFO [StoreOpener-7af30662d67e218e5eed0cf39272545b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7af30662d67e218e5eed0cf39272545b columnFamilyName family 2024-11-24T23:13:32,381 DEBUG [StoreOpener-7af30662d67e218e5eed0cf39272545b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:32,381 INFO [StoreOpener-7af30662d67e218e5eed0cf39272545b-1 {}] regionserver.HStore(327): Store=7af30662d67e218e5eed0cf39272545b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:32,381 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,382 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,382 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,383 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 4120d4bb2e1399a50983e420a0eeded8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68642396, jitterRate=0.022851407527923584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:32,383 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,383 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:32,384 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 4120d4bb2e1399a50983e420a0eeded8: Running coprocessor pre-open hook at 1732490012371Writing region info on filesystem at 1732490012371Initializing all the Stores at 1732490012372 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490012372Cleaning up temporary data from old regions at 1732490012378 (+6 ms)Running coprocessor post-open hooks at 1732490012383 (+5 ms)Region opened successfully at 1732490012383 2024-11-24T23:13:32,384 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,384 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,385 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8., pid=77, masterSystemTime=1732490012329 2024-11-24T23:13:32,386 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,387 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 7af30662d67e218e5eed0cf39272545b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68397045, jitterRate=0.019195392727851868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:32,387 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:32,387 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:32,387 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 7af30662d67e218e5eed0cf39272545b: Running coprocessor pre-open hook at 1732490012378Writing region info on filesystem at 1732490012378Initializing all the Stores at 1732490012379 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490012379Cleaning up temporary data from old regions at 1732490012384 (+5 ms)Running coprocessor post-open hooks at 1732490012387 (+3 ms)Region opened successfully at 1732490012387 2024-11-24T23:13:32,387 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:32,387 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:32,388 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e72e703f9026a43993a24ead31db485, NAME => 'test,iii,1732490008065.6e72e703f9026a43993a24ead31db485.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-24T23:13:32,388 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,388 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,iii,1732490008065.6e72e703f9026a43993a24ead31db485.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:32,388 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b., pid=76, masterSystemTime=1732490012335 2024-11-24T23:13:32,388 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,388 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,390 INFO [StoreOpener-6e72e703f9026a43993a24ead31db485-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,392 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:32,392 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:32,392 INFO [StoreOpener-6e72e703f9026a43993a24ead31db485-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e72e703f9026a43993a24ead31db485 columnFamilyName family 2024-11-24T23:13:32,392 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:32,392 DEBUG [StoreOpener-6e72e703f9026a43993a24ead31db485-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:32,392 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => eb705a3a04d42fd23798b48207c112de, NAME => 'test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-24T23:13:32,393 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,393 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T23:13:32,393 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,393 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,397 INFO [StoreOpener-6e72e703f9026a43993a24ead31db485-1 {}] regionserver.HStore(327): Store=6e72e703f9026a43993a24ead31db485/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:32,398 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,398 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,398 INFO [StoreOpener-eb705a3a04d42fd23798b48207c112de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,400 INFO [StoreOpener-eb705a3a04d42fd23798b48207c112de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb705a3a04d42fd23798b48207c112de columnFamilyName family 2024-11-24T23:13:32,400 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,400 DEBUG [StoreOpener-eb705a3a04d42fd23798b48207c112de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T23:13:32,400 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,400 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,400 INFO [StoreOpener-eb705a3a04d42fd23798b48207c112de-1 {}] regionserver.HStore(327): Store=eb705a3a04d42fd23798b48207c112de/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T23:13:32,401 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,402 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,403 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,403 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,403 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,403 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,404 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 6e72e703f9026a43993a24ead31db485; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71788275, jitterRate=0.06972865760326385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:32,404 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:32,404 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 6e72e703f9026a43993a24ead31db485: Running coprocessor pre-open hook at 1732490012389Writing region info on filesystem at 1732490012389Initializing all the Stores at 1732490012390 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490012390Cleaning up temporary data from old regions at 1732490012400 (+10 ms)Running coprocessor post-open hooks at 1732490012404 (+4 ms)Region opened successfully at 1732490012404 2024-11-24T23:13:32,405 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732490008065.6e72e703f9026a43993a24ead31db485., pid=78, masterSystemTime=1732490012329 2024-11-24T23:13:32,406 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,407 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:32,407 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened eb705a3a04d42fd23798b48207c112de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75404711, jitterRate=0.1236177533864975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T23:13:32,407 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:32,407 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:32,408 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for eb705a3a04d42fd23798b48207c112de: Running coprocessor pre-open hook at 1732490012393Writing region info on filesystem at 1732490012393Initializing all the Stores at 1732490012398 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732490012398Cleaning up temporary data from old regions at 1732490012403 (+5 ms)Running coprocessor post-open hooks at 1732490012407 (+4 ms)Region opened successfully at 1732490012407 2024-11-24T23:13:32,408 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de., pid=75, masterSystemTime=1732490012335 2024-11-24T23:13:32,410 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:32,411 INFO [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:32,420 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,420 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,420 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,421 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,422 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,422 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,422 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,423 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,431 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,432 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,435 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,441 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,442 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,442 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,442 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,442 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,443 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,443 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,443 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,443 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,443 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,443 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,443 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 19 msec 2024-11-24T23:13:32,446 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,446 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,446 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,459 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,459 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,459 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,459 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,459 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,459 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,459 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,459 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN to state=OFFLINE, location=null, table=test, region=f6455d829ffb35ae0e811982e8b77549 to restore RIT 2024-11-24T23:13:32,460 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042460 2024-11-24T23:13:32,460 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,460 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,462 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,464 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,464 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,464 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,45393,1732490005487 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,490 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,493 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,494 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,494 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072494 2024-11-24T23:13:32,494 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132494 2024-11-24T23:13:32,494 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-24T23:13:32,494 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,494 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,494 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,494 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,495 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,495 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,495 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,495 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,496 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,500 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,501 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,501 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,501 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,501 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,501 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,506 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,506 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e72e703f9026a43993a24ead31db485, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,507 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-24T23:13:32,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,508 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,508 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,516 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,520 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,522 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,522 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,522 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,522 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,522 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,522 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,522 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,522 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN to state=OFFLINE, location=null, table=test, region=f6455d829ffb35ae0e811982e8b77549 to restore RIT 2024-11-24T23:13:32,529 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042528 2024-11-24T23:13:32,529 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,529 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,531 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,532 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,532 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,532 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,45393,1732490005487 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,545 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,545 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072545 2024-11-24T23:13:32,545 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132545 2024-11-24T23:13:32,545 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-24T23:13:32,545 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,545 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,545 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,545 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,545 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,545 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,546 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,546 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,552 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,555 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,559 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,559 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,560 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,560 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4120d4bb2e1399a50983e420a0eeded8, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,561 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-24T23:13:32,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,562 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,562 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,573 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,573 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,574 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,574 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,574 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,574 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,574 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,574 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN to state=OFFLINE, location=null, table=test, region=f6455d829ffb35ae0e811982e8b77549 to restore RIT 2024-11-24T23:13:32,575 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042575 2024-11-24T23:13:32,575 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,575 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,576 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,577 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,578 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,578 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,583 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,584 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,585 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,586 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,587 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,589 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072589 2024-11-24T23:13:32,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132589 2024-11-24T23:13:32,589 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-24T23:13:32,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,589 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,589 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,589 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,589 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,590 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,596 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,601 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,602 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7af30662d67e218e5eed0cf39272545b, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,603 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-24T23:13:32,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,604 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,604 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,615 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,615 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,615 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,615 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,616 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN to state=OFFLINE, location=null, table=test, region=f6455d829ffb35ae0e811982e8b77549 to restore RIT 2024-11-24T23:13:32,616 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042616 2024-11-24T23:13:32,616 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,616 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,618 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,619 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,619 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,619 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,630 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,630 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072630 2024-11-24T23:13:32,630 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132630 2024-11-24T23:13:32,630 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-24T23:13:32,631 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,631 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,631 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,631 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,631 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,631 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,631 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,632 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,632 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,636 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,637 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,638 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,643 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,644 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,645 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,646 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb705a3a04d42fd23798b48207c112de, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-24T23:13:32,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,647 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,647 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,657 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,657 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,657 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,658 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,658 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,658 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,658 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,658 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN to state=OFFLINE, location=null, table=test, region=f6455d829ffb35ae0e811982e8b77549 to restore RIT 2024-11-24T23:13:32,658 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042658 2024-11-24T23:13:32,658 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,659 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,661 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,662 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,662 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,662 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,673 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,673 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072673 2024-11-24T23:13:32,673 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132673 2024-11-24T23:13:32,673 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-24T23:13:32,673 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,673 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,673 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,673 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,674 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,674 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,674 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,675 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,678 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,678 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,678 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,678 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,678 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,678 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,678 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,678 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,682 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,683 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,683 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,683 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,683 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,683 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,683 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,683 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,683 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,683 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1244e0d7b7b5b8b2d792af82faee1da6, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,684 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-24T23:13:32,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,685 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,685 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,694 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN to state=OFFLINE, location=null, table=test, region=f6455d829ffb35ae0e811982e8b77549 to restore RIT 2024-11-24T23:13:32,695 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042695 2024-11-24T23:13:32,695 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,695 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,697 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,700 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,700 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,700 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,707 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,707 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072707 2024-11-24T23:13:32,707 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132707 2024-11-24T23:13:32,707 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-24T23:13:32,707 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,707 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,707 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,707 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,707 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,707 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,708 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,708 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,709 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,716 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,717 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,717 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90985da3d485ab013ba37f15b715a5d0, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,717 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-24T23:13:32,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,718 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,719 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN to state=OFFLINE, location=null, table=test, region=f6455d829ffb35ae0e811982e8b77549 to restore RIT 2024-11-24T23:13:32,732 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042732 2024-11-24T23:13:32,732 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,732 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,734 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,735 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,735 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,735 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,741 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,741 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072741 2024-11-24T23:13:32,741 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132741 2024-11-24T23:13:32,741 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T23:13:32,741 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,742 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,742 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,742 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,742 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,742 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,742 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,743 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,747 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,747 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,750 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,750 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,750 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,750 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,750 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,750 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,750 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,751 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,751 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,751 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,751 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,751 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6455d829ffb35ae0e811982e8b77549, server=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,751 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-24T23:13:32,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,752 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,752 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN to state=OFFLINE, location=null, table=test, region=f6455d829ffb35ae0e811982e8b77549 to restore RIT 2024-11-24T23:13:32,763 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042763 2024-11-24T23:13:32,764 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,764 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,765 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,768 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,768 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,768 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,774 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,774 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072774 2024-11-24T23:13:32,774 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132774 2024-11-24T23:13:32,774 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-24T23:13:32,774 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,774 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,774 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,774 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,774 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,775 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,775 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,777 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,778 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,778 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,779 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,779 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,779 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,780 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,781 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,781 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,781 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,781 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,781 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,781 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,781 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,781 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,781 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,781 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,781 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,781 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,781 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,781 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,781 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,781 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,781 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN 2024-11-24T23:13:32,781 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T23:13:32,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,782 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,782 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,793 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,794 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f6455d829ffb35ae0e811982e8b77549, ASSIGN to state=OFFLINE, location=null, table=test, region=f6455d829ffb35ae0e811982e8b77549 to restore RIT 2024-11-24T23:13:32,795 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042795 2024-11-24T23:13:32,795 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,795 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,796 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,797 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,797 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,797 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,800 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,805 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,805 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072805 2024-11-24T23:13:32,805 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132805 2024-11-24T23:13:32,805 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T23:13:32,805 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,805 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,805 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,805 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,805 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,805 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,805 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,806 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,806 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,808 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,809 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,809 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,809 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,809 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,809 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,809 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,809 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,810 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,811 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,811 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,812 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,813 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,813 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,813 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,813 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,813 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,813 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,813 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,813 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,813 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,813 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,813 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,813 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,813 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN 2024-11-24T23:13:32,813 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T23:13:32,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,814 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,814 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,826 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,826 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,826 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,826 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,826 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,826 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,826 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7af30662d67e218e5eed0cf39272545b, ASSIGN to state=OFFLINE, location=null, table=test, region=7af30662d67e218e5eed0cf39272545b to restore RIT 2024-11-24T23:13:32,827 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042827 2024-11-24T23:13:32,827 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,827 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,828 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,832 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,832 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,836 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,836 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,837 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072836 2024-11-24T23:13:32,837 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132837 2024-11-24T23:13:32,837 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T23:13:32,837 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,837 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,837 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,837 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,837 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,837 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,837 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,837 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,838 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,840 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,841 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,842 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,844 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,844 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,844 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,844 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,845 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,845 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,845 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,845 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,845 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,845 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN 2024-11-24T23:13:32,845 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T23:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,846 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,846 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,851 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,857 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,858 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,858 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,858 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4120d4bb2e1399a50983e420a0eeded8, ASSIGN to state=OFFLINE, location=null, table=test, region=4120d4bb2e1399a50983e420a0eeded8 to restore RIT 2024-11-24T23:13:32,858 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042858 2024-11-24T23:13:32,858 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,858 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,859 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,863 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,864 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,865 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,866 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,867 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,867 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,867 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072867 2024-11-24T23:13:32,867 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132867 2024-11-24T23:13:32,868 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T23:13:32,868 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,868 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,868 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,868 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,868 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,868 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,868 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,869 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,869 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,872 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,874 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,875 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,875 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,875 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,875 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,876 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,876 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,876 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN 2024-11-24T23:13:32,876 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T23:13:32,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,877 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,877 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,889 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb705a3a04d42fd23798b48207c112de, ASSIGN to state=OFFLINE, location=null, table=test, region=eb705a3a04d42fd23798b48207c112de to restore RIT 2024-11-24T23:13:32,890 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042890 2024-11-24T23:13:32,890 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,890 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,891 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,892 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,892 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,892 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,898 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,898 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072898 2024-11-24T23:13:32,898 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132898 2024-11-24T23:13:32,898 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-24T23:13:32,898 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,898 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,898 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,898 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,898 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,898 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,899 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,899 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,900 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,901 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,902 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,904 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,905 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,905 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,905 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,905 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN 2024-11-24T23:13:32,905 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T23:13:32,905 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,906 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,906 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=792edfc226c30220c15d3c2a8c9ffa60, ASSIGN to state=OFFLINE, location=null, table=test, region=792edfc226c30220c15d3c2a8c9ffa60 to restore RIT 2024-11-24T23:13:32,916 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042916 2024-11-24T23:13:32,916 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,916 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,917 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,918 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,918 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,918 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,923 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,923 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072923 2024-11-24T23:13:32,923 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132923 2024-11-24T23:13:32,923 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-24T23:13:32,923 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,924 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,924 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,924 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,924 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,924 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,924 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,924 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,925 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,927 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,929 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,929 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN 2024-11-24T23:13:32,929 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T23:13:32,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,930 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,930 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=90985da3d485ab013ba37f15b715a5d0, ASSIGN to state=OFFLINE, location=null, table=test, region=90985da3d485ab013ba37f15b715a5d0 to restore RIT 2024-11-24T23:13:32,942 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042942 2024-11-24T23:13:32,942 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,942 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,943 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:32,945 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:32,945 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,945 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:32,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:32,950 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:32,950 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490072950 2024-11-24T23:13:32,950 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490132950 2024-11-24T23:13:32,950 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-24T23:13:32,950 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:32,950 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:32,950 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:32,950 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:32,950 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:32,950 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:32,950 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:32,951 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:32,951 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:32,953 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:32,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:32,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:32,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:32,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:32,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN 2024-11-24T23:13:32,955 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T23:13:32,956 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:32,956 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:32,956 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:32,959 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:32,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:32,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:32,998 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:32,998 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1244e0d7b7b5b8b2d792af82faee1da6, ASSIGN to state=OFFLINE, location=null, table=test, region=1244e0d7b7b5b8b2d792af82faee1da6 to restore RIT 2024-11-24T23:13:32,999 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490042999 2024-11-24T23:13:32,999 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:32,999 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,000 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:33,001 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:33,001 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:33,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,001 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,008 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:33,008 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490073008 2024-11-24T23:13:33,008 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490133008 2024-11-24T23:13:33,008 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T23:13:33,008 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:33,008 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,008 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:33,008 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:33,009 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:33,009 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:33,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:33,009 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:33,010 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:33,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:33,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:33,012 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,013 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:33,013 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:33,013 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:33,013 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,013 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:33,013 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:33,014 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:33,015 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,016 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,016 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:33,017 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN 2024-11-24T23:13:33,017 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T23:13:33,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:33,018 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:33,018 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:33,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=6e72e703f9026a43993a24ead31db485, ASSIGN to state=OFFLINE, location=null, table=test, region=6e72e703f9026a43993a24ead31db485 to restore RIT 2024-11-24T23:13:33,041 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490043041 2024-11-24T23:13:33,041 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:33,041 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:33,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:33,044 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:33,049 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490073049 2024-11-24T23:13:33,049 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490133049 2024-11-24T23:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-24T23:13:33,049 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:33,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,049 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:33,049 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:33,050 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:33,050 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:33,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:33,050 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:33,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:33,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:33,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:33,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:33,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:33,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:33,052 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:33,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:33,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:33,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,054 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T23:13:33,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:33,055 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:33,055 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:33,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,067 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,067 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,067 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,068 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490043068 2024-11-24T23:13:33,068 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:33,068 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,070 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:33,071 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:33,071 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,071 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,074 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,077 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:33,077 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490073077 2024-11-24T23:13:33,077 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490133077 2024-11-24T23:13:33,077 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-24T23:13:33,077 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:33,077 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,077 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:33,077 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:33,078 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:33,078 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:33,078 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:33,078 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:33,079 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:33,081 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:33,081 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,081 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:33,081 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:33,081 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:33,081 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:33,082 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:33,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:33,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:33,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:33,083 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,083 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:33,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,084 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.1732490007246, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,084 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T23:13:33,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:33,085 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:33,085 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:33,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,093 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,093 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,093 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,094 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490043094 2024-11-24T23:13:33,094 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:33,094 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,096 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:33,097 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:33,097 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:33,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,097 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,103 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:33,103 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490073103 2024-11-24T23:13:33,103 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490133103 2024-11-24T23:13:33,103 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-24T23:13:33,103 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:33,103 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,103 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:33,103 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:33,103 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:33,103 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:33,103 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:33,104 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:33,104 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:33,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:33,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:33,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:33,106 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T23:13:33,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:33,107 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 2024-11-24T23:13:33,107 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,108 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,37739,1732490005584 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,108 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T23:13:33,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:33,108 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:33,109 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:33,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,120 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,121 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490043120 2024-11-24T23:13:33,121 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:33,121 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,122 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:33,123 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:33,123 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:33,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,123 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,128 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:33,128 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490073128 2024-11-24T23:13:33,128 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490133128 2024-11-24T23:13:33,128 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-24T23:13:33,128 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:33,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,128 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:33,128 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:33,129 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:33,129 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:33,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:33,129 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:33,130 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:33,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:33,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:33,131 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T23:13:33,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:33,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:33,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:33,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,132 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T23:13:33,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:33,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-24T23:13:33,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:33,133 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:33,133 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:33,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,146 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490043146 2024-11-24T23:13:33,147 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:33,147 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,148 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:33,149 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:33,149 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,149 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,153 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,154 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:33,154 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490073154 2024-11-24T23:13:33,154 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490133154 2024-11-24T23:13:33,154 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-24T23:13:33,154 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:33,154 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,154 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:33,154 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:33,155 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:33,155 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:33,155 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:33,155 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:33,156 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:33,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:33,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T23:13:33,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:33,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:33,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:33,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,158 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:33,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,158 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-24T23:13:33,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:33,159 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:33,159 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:33,172 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T23:13:33,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,173 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490043173 2024-11-24T23:13:33,173 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:33,173 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,174 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:33,175 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:33,175 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,176 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,180 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,182 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:33,182 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490073182 2024-11-24T23:13:33,182 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490133182 2024-11-24T23:13:33,182 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-24T23:13:33,182 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:33,182 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,182 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:33,182 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:33,182 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:33,182 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:33,182 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:33,182 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:33,183 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:33,185 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:33,185 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T23:13:33,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:33,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:33,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:33,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,186 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:33,186 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure ba7f32532130%2C36885%2C1732490005631.meta.1732490007620.meta, worker=ba7f32532130,45393,1732490005487 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T23:13:33,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T23:13:33,187 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:33,187 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:33,187 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:33,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,205 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490043204 2024-11-24T23:13:33,205 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:33,206 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,208 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:33,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T23:13:33,209 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:33,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,210 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,213 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:33,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490073213 2024-11-24T23:13:33,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490133213 2024-11-24T23:13:33,213 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-24T23:13:33,213 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T23:13:33,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,213 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:33,214 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:33,214 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T23:13:33,214 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T23:13:33,214 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T23:13:33,214 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T23:13:33,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T23:13:33,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T23:13:33,217 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T23:13:33,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T23:13:33,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true 2024-11-24T23:13:33,218 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T23:13:33,218 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T23:13:33,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=61 2024-11-24T23:13:33,219 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=ba7f32532130,37739,1732490005584, table=hbase:meta, region=1588230740 2024-11-24T23:13:33,219 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,37739,1732490005584, state=OPEN 2024-11-24T23:13:33,223 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34129 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T23:13:33,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T23:13:33,231 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732490043231 2024-11-24T23:13:33,231 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T23:13:33,231 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T23:13:33,233 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T23:13:33,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 792edfc226c30220c15d3c2a8c9ffa60, server=ba7f32532130,45393,1732490005487:java.lang.RuntimeException: inject code bug; ServerCrashProcedure ba7f32532130,36885,1732490005631, splitWal=true, meta=true exec-time=2.6540 sec 2024-11-24T23:13:33,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afc068dc7ba93a54eff038a8f06c8542, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4120d4bb2e1399a50983e420a0eeded8, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4c5d46f0872640f4da48af2d12d1131, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ee88c2c22e3a06388845a14a86f9b2, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0ab12bee6921e08663b5265ccf589a9d, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb705a3a04d42fd23798b48207c112de, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79f30edc5579a1fb86c63410b3b6d59a, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=740a2018f9a8f09b78987d38b376567b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e72e703f9026a43993a24ead31db485, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97ef64c446508710b8b41414b50b97d8, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df65ea81e19f0ed6cb718601a302e9ef, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7af30662d67e218e5eed0cf39272545b, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32141aa52f6862fa21af187e3ac3b72, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69fc34e6caf2036ee015474372306228, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=792edfc226c30220c15d3c2a8c9ffa60, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a817676ac0dff05fc27e6ec9f8e6c89, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a1389df947c85301aa553a546616498, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1244e0d7b7b5b8b2d792af82faee1da6, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874a4cd5ca2cb048061687443fd87c9, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c50f7d0b8194465f687d68a15e4d9d13, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f6455d829ffb35ae0e811982e8b77549, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5362f4dec21f673244ee873440d1779, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64743c3cc4326247d7ff0a96361c7e49, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90985da3d485ab013ba37f15b715a5d0, regionState=OPENING, lastHost=ba7f32532130,36885,1732490005631, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f0e42305404ba59b5243b2bf8292cf0, regionState=OPEN, lastHost=ba7f32532130,45393,1732490005487, regionLocation=ba7f32532130,45393,1732490005487, openSeqNum=2 2024-11-24T23:13:33,238 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adb73b3b244f40e15cbf9254ee22925b, regionState=OPEN, lastHost=ba7f32532130,37739,1732490005584, regionLocation=ba7f32532130,37739,1732490005584, openSeqNum=2 2024-11-24T23:13:33,238 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T23:13:33,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732490073238 2024-11-24T23:13:33,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732490133238 2024-11-24T23:13:33,238 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-24T23:13:33,241 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-24T23:13:33,254 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=352 (was 386), OpenFileDescriptor=622 (was 645), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=295 (was 295), ProcessCount=11 (was 11), AvailableMemoryMB=7320 (was 7365) 2024-11-24T23:13:33,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T23:13:33,255 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T23:13:33,255 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T23:13:33,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:33,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:33,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T23:13:33,256 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T23:13:33,256 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=546622927, stopped=false 2024-11-24T23:13:33,256 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba7f32532130,34129,1732490004717 2024-11-24T23:13:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T23:13:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T23:13:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T23:13:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:33,267 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,267 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T23:13:33,268 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T23:13:33,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:33,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T23:13:33,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T23:13:33,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T23:13:33,268 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,45393,1732490005487' ***** 2024-11-24T23:13:33,268 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T23:13:33,268 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,37739,1732490005584' ***** 2024-11-24T23:13:33,268 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T23:13:33,268 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T23:13:33,269 INFO [RS:1;ba7f32532130:37739 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T23:13:33,269 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T23:13:33,269 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T23:13:33,269 INFO [RS:1;ba7f32532130:37739 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T23:13:33,269 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T23:13:33,269 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for 90985da3d485ab013ba37f15b715a5d0 2024-11-24T23:13:33,269 INFO [RS:0;ba7f32532130:45393 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T23:13:33,269 INFO [RS:0;ba7f32532130:45393 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T23:13:33,269 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 792edfc226c30220c15d3c2a8c9ffa60 2024-11-24T23:13:33,269 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 6e72e703f9026a43993a24ead31db485 2024-11-24T23:13:33,269 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 7a1389df947c85301aa553a546616498 2024-11-24T23:13:33,269 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for adb73b3b244f40e15cbf9254ee22925b 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 79f30edc5579a1fb86c63410b3b6d59a 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for df65ea81e19f0ed6cb718601a302e9ef 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for 6a817676ac0dff05fc27e6ec9f8e6c89 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 2f0e42305404ba59b5243b2bf8292cf0 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for 6874a4cd5ca2cb048061687443fd87c9 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for 1244e0d7b7b5b8b2d792af82faee1da6 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for afc068dc7ba93a54eff038a8f06c8542 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for 740a2018f9a8f09b78987d38b376567b 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for c50f7d0b8194465f687d68a15e4d9d13 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for b4c5d46f0872640f4da48af2d12d1131 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 64743c3cc4326247d7ff0a96361c7e49 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for c8ee88c2c22e3a06388845a14a86f9b2 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 97ef64c446508710b8b41414b50b97d8 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for b32141aa52f6862fa21af187e3ac3b72 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ab12bee6921e08663b5265ccf589a9d 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for 7af30662d67e218e5eed0cf39272545b 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 69fc34e6caf2036ee015474372306228 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(3091): Received CLOSE for 4120d4bb2e1399a50983e420a0eeded8 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for a5362f4dec21f673244ee873440d1779 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,37739,1732490005584 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T23:13:33,270 INFO [RS:1;ba7f32532130:37739 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ba7f32532130:37739. 2024-11-24T23:13:33,270 INFO [RS:0;ba7f32532130:45393 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba7f32532130:45393. 2024-11-24T23:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 90985da3d485ab013ba37f15b715a5d0, disabling compactions & flushes 2024-11-24T23:13:33,270 DEBUG [RS:1;ba7f32532130:37739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T23:13:33,270 DEBUG [RS:0;ba7f32532130:45393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T23:13:33,270 DEBUG [RS:1;ba7f32532130:37739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:33,270 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:33,270 DEBUG [RS:0;ba7f32532130:45393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:33,271 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-24T23:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 792edfc226c30220c15d3c2a8c9ffa60, disabling compactions & flushes 2024-11-24T23:13:33,271 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:33,271 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:33,271 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. after waiting 0 ms 2024-11-24T23:13:33,271 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:33,271 DEBUG [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(1351): Waiting on 0ab12bee6921e08663b5265ccf589a9d, 2f0e42305404ba59b5243b2bf8292cf0, 4120d4bb2e1399a50983e420a0eeded8, 64743c3cc4326247d7ff0a96361c7e49, 69fc34e6caf2036ee015474372306228, 6e72e703f9026a43993a24ead31db485, 792edfc226c30220c15d3c2a8c9ffa60, 79f30edc5579a1fb86c63410b3b6d59a, 7a1389df947c85301aa553a546616498, 97ef64c446508710b8b41414b50b97d8, afc068dc7ba93a54eff038a8f06c8542, c50f7d0b8194465f687d68a15e4d9d13, f6455d829ffb35ae0e811982e8b77549 2024-11-24T23:13:33,271 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:33,271 INFO [RS:1;ba7f32532130:37739 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T23:13:33,271 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. after waiting 0 ms 2024-11-24T23:13:33,271 INFO [RS:1;ba7f32532130:37739 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T23:13:33,271 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:33,271 INFO [RS:1;ba7f32532130:37739 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T23:13:33,271 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T23:13:33,272 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-24T23:13:33,272 DEBUG [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(1351): Waiting on 1244e0d7b7b5b8b2d792af82faee1da6, 1588230740, 6874a4cd5ca2cb048061687443fd87c9, 6a817676ac0dff05fc27e6ec9f8e6c89, 740a2018f9a8f09b78987d38b376567b, 7af30662d67e218e5eed0cf39272545b, 90985da3d485ab013ba37f15b715a5d0, a5362f4dec21f673244ee873440d1779, adb73b3b244f40e15cbf9254ee22925b, b32141aa52f6862fa21af187e3ac3b72, b4c5d46f0872640f4da48af2d12d1131, c8ee88c2c22e3a06388845a14a86f9b2, df65ea81e19f0ed6cb718601a302e9ef, eb705a3a04d42fd23798b48207c112de 2024-11-24T23:13:33,272 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T23:13:33,272 INFO [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T23:13:33,272 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T23:13:33,272 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T23:13:33,272 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T23:13:33,272 INFO [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-24T23:13:33,277 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/90985da3d485ab013ba37f15b715a5d0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,277 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/792edfc226c30220c15d3c2a8c9ffa60/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,279 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:33,279 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:33,279 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 792edfc226c30220c15d3c2a8c9ffa60: Waiting for close lock at 1732490013270Running coprocessor pre-close hooks at 1732490013270Disabling compacts and flushes for region at 1732490013270Disabling writes for close at 1732490013271 (+1 ms)Writing region close event to WAL at 1732490013272 (+1 ms)Running coprocessor post-close hooks at 1732490013278 (+6 ms)Closed at 1732490013279 (+1 ms) 2024-11-24T23:13:33,279 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 90985da3d485ab013ba37f15b715a5d0: Waiting for close lock at 1732490013270Running coprocessor pre-close hooks at 1732490013270Disabling compacts and flushes for region at 1732490013270Disabling writes for close at 1732490013271 (+1 ms)Writing region close event to WAL at 1732490013272 (+1 ms)Running coprocessor post-close hooks at 1732490013277 (+5 ms)Closed at 1732490013279 (+2 ms) 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732490008065.90985da3d485ab013ba37f15b715a5d0. 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732490008065.792edfc226c30220c15d3c2a8c9ffa60. 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing adb73b3b244f40e15cbf9254ee22925b, disabling compactions & flushes 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6e72e703f9026a43993a24ead31db485, disabling compactions & flushes 2024-11-24T23:13:33,280 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:33,280 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. after waiting 0 ms 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. after waiting 0 ms 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:33,280 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:33,282 INFO [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T23:13:33,283 INFO [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T23:13:33,286 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6e72e703f9026a43993a24ead31db485/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,287 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:33,287 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6e72e703f9026a43993a24ead31db485: Waiting for close lock at 1732490013280Running coprocessor pre-close hooks at 1732490013280Disabling compacts and flushes for region at 1732490013280Disabling writes for close at 1732490013280Writing region close event to WAL at 1732490013281 (+1 ms)Running coprocessor post-close hooks at 1732490013287 (+6 ms)Closed at 1732490013287 2024-11-24T23:13:33,287 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732490008065.6e72e703f9026a43993a24ead31db485. 2024-11-24T23:13:33,288 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7a1389df947c85301aa553a546616498, disabling compactions & flushes 2024-11-24T23:13:33,288 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:33,288 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:33,288 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732490008065.7a1389df947c85301aa553a546616498. after waiting 0 ms 2024-11-24T23:13:33,288 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:33,288 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/info/cd3b76bcb8e04af486f31b708c594a3e is 138, key is test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8./info:regioninfo/1732490012064/Put/seqid=0 2024-11-24T23:13:33,288 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/adb73b3b244f40e15cbf9254ee22925b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,289 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:33,289 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for adb73b3b244f40e15cbf9254ee22925b: Waiting for close lock at 1732490013280Running coprocessor pre-close hooks at 1732490013280Disabling compacts and flushes for region at 1732490013280Disabling writes for close at 1732490013280Writing region close event to WAL at 1732490013281 (+1 ms)Running coprocessor post-close hooks at 1732490013289 (+8 ms)Closed at 1732490013289 2024-11-24T23:13:33,289 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732490008065.adb73b3b244f40e15cbf9254ee22925b. 2024-11-24T23:13:33,289 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df65ea81e19f0ed6cb718601a302e9ef, disabling compactions & flushes 2024-11-24T23:13:33,290 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:33,290 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:33,290 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. after waiting 0 ms 2024-11-24T23:13:33,290 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:33,293 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7a1389df947c85301aa553a546616498/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,294 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7a1389df947c85301aa553a546616498: Waiting for close lock at 1732490013288Running coprocessor pre-close hooks at 1732490013288Disabling compacts and flushes for region at 1732490013288Disabling writes for close at 1732490013288Writing region close event to WAL at 1732490013288Running coprocessor post-close hooks at 1732490013294 (+6 ms)Closed at 1732490013294 2024-11-24T23:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732490008065.7a1389df947c85301aa553a546616498. 2024-11-24T23:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 79f30edc5579a1fb86c63410b3b6d59a, disabling compactions & flushes 2024-11-24T23:13:33,294 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:33,295 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. after waiting 0 ms 2024-11-24T23:13:33,295 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:33,297 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/df65ea81e19f0ed6cb718601a302e9ef/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,298 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:33,298 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df65ea81e19f0ed6cb718601a302e9ef: Waiting for close lock at 1732490013289Running coprocessor pre-close hooks at 1732490013289Disabling compacts and flushes for region at 1732490013289Disabling writes for close at 1732490013290 (+1 ms)Writing region close event to WAL at 1732490013291 (+1 ms)Running coprocessor post-close hooks at 1732490013298 (+7 ms)Closed at 1732490013298 2024-11-24T23:13:33,299 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732490008065.df65ea81e19f0ed6cb718601a302e9ef. 2024-11-24T23:13:33,299 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6a817676ac0dff05fc27e6ec9f8e6c89, disabling compactions & flushes 2024-11-24T23:13:33,299 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:33,299 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:33,299 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. after waiting 0 ms 2024-11-24T23:13:33,299 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:33,305 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/79f30edc5579a1fb86c63410b3b6d59a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741869_1045 (size=8195) 2024-11-24T23:13:33,307 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:33,307 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 79f30edc5579a1fb86c63410b3b6d59a: Waiting for close lock at 1732490013294Running coprocessor pre-close hooks at 1732490013294Disabling compacts and flushes for region at 1732490013294Disabling writes for close at 1732490013295 (+1 ms)Writing region close event to WAL at 1732490013296 (+1 ms)Running coprocessor post-close hooks at 1732490013307 (+11 ms)Closed at 1732490013307 2024-11-24T23:13:33,307 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732490008065.79f30edc5579a1fb86c63410b3b6d59a. 2024-11-24T23:13:33,307 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2f0e42305404ba59b5243b2bf8292cf0, disabling compactions & flushes 2024-11-24T23:13:33,307 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:33,307 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:33,308 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. after waiting 0 ms 2024-11-24T23:13:33,308 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:33,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741869_1045 (size=8195) 2024-11-24T23:13:33,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741869_1045 (size=8195) 2024-11-24T23:13:33,310 INFO [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/info/cd3b76bcb8e04af486f31b708c594a3e 2024-11-24T23:13:33,317 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6a817676ac0dff05fc27e6ec9f8e6c89/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/2f0e42305404ba59b5243b2bf8292cf0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,318 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6a817676ac0dff05fc27e6ec9f8e6c89: Waiting for close lock at 1732490013299Running coprocessor pre-close hooks at 1732490013299Disabling compacts and flushes for region at 1732490013299Disabling writes for close at 1732490013299Writing region close event to WAL at 1732490013304 (+5 ms)Running coprocessor post-close hooks at 1732490013318 (+14 ms)Closed at 1732490013318 2024-11-24T23:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732490008065.6a817676ac0dff05fc27e6ec9f8e6c89. 2024-11-24T23:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6874a4cd5ca2cb048061687443fd87c9, disabling compactions & flushes 2024-11-24T23:13:33,318 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:33,318 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2f0e42305404ba59b5243b2bf8292cf0: Waiting for close lock at 1732490013307Running coprocessor pre-close hooks at 1732490013307Disabling compacts and flushes for region at 1732490013307Disabling writes for close at 1732490013308 (+1 ms)Writing region close event to WAL at 1732490013309 (+1 ms)Running coprocessor post-close hooks at 1732490013318 (+9 ms)Closed at 1732490013318 2024-11-24T23:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. after waiting 0 ms 2024-11-24T23:13:33,319 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732490008065.2f0e42305404ba59b5243b2bf8292cf0. 2024-11-24T23:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:33,319 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f6455d829ffb35ae0e811982e8b77549, disabling compactions & flushes 2024-11-24T23:13:33,319 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:33,319 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:33,319 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. after waiting 0 ms 2024-11-24T23:13:33,319 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:33,320 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/.tmp/info/cd3b76bcb8e04af486f31b708c594a3e as hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/info/cd3b76bcb8e04af486f31b708c594a3e 2024-11-24T23:13:33,325 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/f6455d829ffb35ae0e811982e8b77549/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,326 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:33,326 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f6455d829ffb35ae0e811982e8b77549: Waiting for close lock at 1732490013319Running coprocessor pre-close hooks at 1732490013319Disabling compacts and flushes for region at 1732490013319Disabling writes for close at 1732490013319Writing region close event to WAL at 1732490013320 (+1 ms)Running coprocessor post-close hooks at 1732490013326 (+6 ms)Closed at 1732490013326 2024-11-24T23:13:33,326 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732490008065.f6455d829ffb35ae0e811982e8b77549. 2024-11-24T23:13:33,326 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing afc068dc7ba93a54eff038a8f06c8542, disabling compactions & flushes 2024-11-24T23:13:33,327 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:33,327 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:33,327 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. after waiting 0 ms 2024-11-24T23:13:33,327 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:33,328 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/6874a4cd5ca2cb048061687443fd87c9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,329 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:33,330 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6874a4cd5ca2cb048061687443fd87c9: Waiting for close lock at 1732490013318Running coprocessor pre-close hooks at 1732490013318Disabling compacts and flushes for region at 1732490013318Disabling writes for close at 1732490013318Writing region close event to WAL at 1732490013319 (+1 ms)Running coprocessor post-close hooks at 1732490013329 (+10 ms)Closed at 1732490013329 2024-11-24T23:13:33,330 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732490008065.6874a4cd5ca2cb048061687443fd87c9. 2024-11-24T23:13:33,330 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1244e0d7b7b5b8b2d792af82faee1da6, disabling compactions & flushes 2024-11-24T23:13:33,330 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:33,330 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:33,330 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. after waiting 0 ms 2024-11-24T23:13:33,330 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:33,333 INFO [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/info/cd3b76bcb8e04af486f31b708c594a3e, entries=24, sequenceid=71, filesize=8.0 K 2024-11-24T23:13:33,333 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/afc068dc7ba93a54eff038a8f06c8542/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,334 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:33,334 INFO [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 62ms, sequenceid=71, compaction requested=false 2024-11-24T23:13:33,334 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for afc068dc7ba93a54eff038a8f06c8542: Waiting for close lock at 1732490013326Running coprocessor pre-close hooks at 1732490013326Disabling compacts and flushes for region at 1732490013326Disabling writes for close at 1732490013327 (+1 ms)Writing region close event to WAL at 1732490013327Running coprocessor post-close hooks at 1732490013334 (+7 ms)Closed at 1732490013334 2024-11-24T23:13:33,335 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732490008065.afc068dc7ba93a54eff038a8f06c8542. 2024-11-24T23:13:33,335 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c50f7d0b8194465f687d68a15e4d9d13, disabling compactions & flushes 2024-11-24T23:13:33,335 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:33,335 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:33,335 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. after waiting 0 ms 2024-11-24T23:13:33,335 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:33,336 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/1244e0d7b7b5b8b2d792af82faee1da6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,336 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:33,337 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1244e0d7b7b5b8b2d792af82faee1da6: Waiting for close lock at 1732490013330Running coprocessor pre-close hooks at 1732490013330Disabling compacts and flushes for region at 1732490013330Disabling writes for close at 1732490013330Writing region close event to WAL at 1732490013331 (+1 ms)Running coprocessor post-close hooks at 1732490013336 (+5 ms)Closed at 1732490013336 2024-11-24T23:13:33,337 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732490008065.1244e0d7b7b5b8b2d792af82faee1da6. 2024-11-24T23:13:33,337 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 740a2018f9a8f09b78987d38b376567b, disabling compactions & flushes 2024-11-24T23:13:33,337 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:33,337 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:33,337 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. after waiting 0 ms 2024-11-24T23:13:33,337 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:33,340 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/c50f7d0b8194465f687d68a15e4d9d13/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,341 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-24T23:13:33,342 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:33,342 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c50f7d0b8194465f687d68a15e4d9d13: Waiting for close lock at 1732490013335Running coprocessor pre-close hooks at 1732490013335Disabling compacts and flushes for region at 1732490013335Disabling writes for close at 1732490013335Writing region close event to WAL at 1732490013335Running coprocessor post-close hooks at 1732490013341 (+6 ms)Closed at 1732490013342 (+1 ms) 2024-11-24T23:13:33,342 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732490008065.c50f7d0b8194465f687d68a15e4d9d13. 2024-11-24T23:13:33,342 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T23:13:33,342 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 64743c3cc4326247d7ff0a96361c7e49, disabling compactions & flushes 2024-11-24T23:13:33,342 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:33,342 INFO [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T23:13:33,342 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:33,342 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. after waiting 0 ms 2024-11-24T23:13:33,342 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:33,342 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732490013272Running coprocessor pre-close hooks at 1732490013272Disabling compacts and flushes for region at 1732490013272Disabling writes for close at 1732490013272Obtaining lock to block concurrent updates at 1732490013272Preparing flush snapshotting stores in 1588230740 at 1732490013272Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732490013280 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732490013282 (+2 ms)Flushing 1588230740/info: creating writer at 1732490013282Flushing 1588230740/info: appending metadata at 1732490013287 (+5 ms)Flushing 1588230740/info: closing flushed file at 1732490013287Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d1953d: reopening flushed file at 1732490013319 (+32 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 62ms, sequenceid=71, compaction requested=false at 1732490013334 (+15 ms)Writing region close event to WAL at 1732490013337 (+3 ms)Running coprocessor post-close hooks at 1732490013342 (+5 ms)Closed at 1732490013342 2024-11-24T23:13:33,343 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T23:13:33,344 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/740a2018f9a8f09b78987d38b376567b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,345 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:33,345 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 740a2018f9a8f09b78987d38b376567b: Waiting for close lock at 1732490013337Running coprocessor pre-close hooks at 1732490013337Disabling compacts and flushes for region at 1732490013337Disabling writes for close at 1732490013337Writing region close event to WAL at 1732490013338 (+1 ms)Running coprocessor post-close hooks at 1732490013345 (+7 ms)Closed at 1732490013345 2024-11-24T23:13:33,345 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732490008065.740a2018f9a8f09b78987d38b376567b. 2024-11-24T23:13:33,345 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b4c5d46f0872640f4da48af2d12d1131, disabling compactions & flushes 2024-11-24T23:13:33,345 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:33,345 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:33,345 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. after waiting 0 ms 2024-11-24T23:13:33,345 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:33,348 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/64743c3cc4326247d7ff0a96361c7e49/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,348 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 64743c3cc4326247d7ff0a96361c7e49: Waiting for close lock at 1732490013342Running coprocessor pre-close hooks at 1732490013342Disabling compacts and flushes for region at 1732490013342Disabling writes for close at 1732490013342Writing region close event to WAL at 1732490013343 (+1 ms)Running coprocessor post-close hooks at 1732490013348 (+5 ms)Closed at 1732490013348 2024-11-24T23:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732490008065.64743c3cc4326247d7ff0a96361c7e49. 2024-11-24T23:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 97ef64c446508710b8b41414b50b97d8, disabling compactions & flushes 2024-11-24T23:13:33,349 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. after waiting 0 ms 2024-11-24T23:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:33,350 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/b4c5d46f0872640f4da48af2d12d1131/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,351 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:33,351 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b4c5d46f0872640f4da48af2d12d1131: Waiting for close lock at 1732490013345Running coprocessor pre-close hooks at 1732490013345Disabling compacts and flushes for region at 1732490013345Disabling writes for close at 1732490013345Writing region close event to WAL at 1732490013346 (+1 ms)Running coprocessor post-close hooks at 1732490013351 (+5 ms)Closed at 1732490013351 2024-11-24T23:13:33,351 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732490008065.b4c5d46f0872640f4da48af2d12d1131. 2024-11-24T23:13:33,351 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c8ee88c2c22e3a06388845a14a86f9b2, disabling compactions & flushes 2024-11-24T23:13:33,352 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:33,352 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:33,352 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. after waiting 0 ms 2024-11-24T23:13:33,352 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:33,354 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/97ef64c446508710b8b41414b50b97d8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,354 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:33,354 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 97ef64c446508710b8b41414b50b97d8: Waiting for close lock at 1732490013349Running coprocessor pre-close hooks at 1732490013349Disabling compacts and flushes for region at 1732490013349Disabling writes for close at 1732490013349Writing region close event to WAL at 1732490013350 (+1 ms)Running coprocessor post-close hooks at 1732490013354 (+4 ms)Closed at 1732490013354 2024-11-24T23:13:33,355 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732490008065.97ef64c446508710b8b41414b50b97d8. 2024-11-24T23:13:33,355 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ab12bee6921e08663b5265ccf589a9d, disabling compactions & flushes 2024-11-24T23:13:33,355 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:33,355 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:33,355 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. after waiting 0 ms 2024-11-24T23:13:33,355 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:33,357 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/c8ee88c2c22e3a06388845a14a86f9b2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,357 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:33,357 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c8ee88c2c22e3a06388845a14a86f9b2: Waiting for close lock at 1732490013351Running coprocessor pre-close hooks at 1732490013351Disabling compacts and flushes for region at 1732490013351Disabling writes for close at 1732490013352 (+1 ms)Writing region close event to WAL at 1732490013352Running coprocessor post-close hooks at 1732490013357 (+5 ms)Closed at 1732490013357 2024-11-24T23:13:33,358 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732490008065.c8ee88c2c22e3a06388845a14a86f9b2. 2024-11-24T23:13:33,358 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b32141aa52f6862fa21af187e3ac3b72, disabling compactions & flushes 2024-11-24T23:13:33,358 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:33,358 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:33,358 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. after waiting 0 ms 2024-11-24T23:13:33,358 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:33,360 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/0ab12bee6921e08663b5265ccf589a9d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,360 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:33,361 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ab12bee6921e08663b5265ccf589a9d: Waiting for close lock at 1732490013355Running coprocessor pre-close hooks at 1732490013355Disabling compacts and flushes for region at 1732490013355Disabling writes for close at 1732490013355Writing region close event to WAL at 1732490013355Running coprocessor post-close hooks at 1732490013360 (+5 ms)Closed at 1732490013360 2024-11-24T23:13:33,361 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732490008065.0ab12bee6921e08663b5265ccf589a9d. 2024-11-24T23:13:33,361 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 69fc34e6caf2036ee015474372306228, disabling compactions & flushes 2024-11-24T23:13:33,361 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:33,361 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:33,361 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. after waiting 0 ms 2024-11-24T23:13:33,361 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:33,362 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/b32141aa52f6862fa21af187e3ac3b72/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,363 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:33,363 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b32141aa52f6862fa21af187e3ac3b72: Waiting for close lock at 1732490013358Running coprocessor pre-close hooks at 1732490013358Disabling compacts and flushes for region at 1732490013358Disabling writes for close at 1732490013358Writing region close event to WAL at 1732490013358Running coprocessor post-close hooks at 1732490013363 (+5 ms)Closed at 1732490013363 2024-11-24T23:13:33,363 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732490008065.b32141aa52f6862fa21af187e3ac3b72. 2024-11-24T23:13:33,364 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7af30662d67e218e5eed0cf39272545b, disabling compactions & flushes 2024-11-24T23:13:33,364 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:33,364 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:33,364 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. after waiting 0 ms 2024-11-24T23:13:33,364 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:33,365 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/69fc34e6caf2036ee015474372306228/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,366 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:33,366 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 69fc34e6caf2036ee015474372306228: Waiting for close lock at 1732490013361Running coprocessor pre-close hooks at 1732490013361Disabling compacts and flushes for region at 1732490013361Disabling writes for close at 1732490013361Writing region close event to WAL at 1732490013362 (+1 ms)Running coprocessor post-close hooks at 1732490013366 (+4 ms)Closed at 1732490013366 2024-11-24T23:13:33,366 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732490008065.69fc34e6caf2036ee015474372306228. 2024-11-24T23:13:33,367 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4120d4bb2e1399a50983e420a0eeded8, disabling compactions & flushes 2024-11-24T23:13:33,367 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:33,367 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:33,367 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. after waiting 0 ms 2024-11-24T23:13:33,367 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:33,368 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/7af30662d67e218e5eed0cf39272545b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,369 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:33,369 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7af30662d67e218e5eed0cf39272545b: Waiting for close lock at 1732490013364Running coprocessor pre-close hooks at 1732490013364Disabling compacts and flushes for region at 1732490013364Disabling writes for close at 1732490013364Writing region close event to WAL at 1732490013364Running coprocessor post-close hooks at 1732490013369 (+5 ms)Closed at 1732490013369 2024-11-24T23:13:33,369 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732490008065.7af30662d67e218e5eed0cf39272545b. 2024-11-24T23:13:33,369 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb705a3a04d42fd23798b48207c112de, disabling compactions & flushes 2024-11-24T23:13:33,370 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:33,370 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:33,370 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. after waiting 0 ms 2024-11-24T23:13:33,370 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:33,371 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/4120d4bb2e1399a50983e420a0eeded8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,373 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:33,373 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4120d4bb2e1399a50983e420a0eeded8: Waiting for close lock at 1732490013367Running coprocessor pre-close hooks at 1732490013367Disabling compacts and flushes for region at 1732490013367Disabling writes for close at 1732490013367Writing region close event to WAL at 1732490013367Running coprocessor post-close hooks at 1732490013372 (+5 ms)Closed at 1732490013372 2024-11-24T23:13:33,373 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732490008065.4120d4bb2e1399a50983e420a0eeded8. 2024-11-24T23:13:33,375 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/eb705a3a04d42fd23798b48207c112de/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,375 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:33,375 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb705a3a04d42fd23798b48207c112de: Waiting for close lock at 1732490013369Running coprocessor pre-close hooks at 1732490013369Disabling compacts and flushes for region at 1732490013369Disabling writes for close at 1732490013370 (+1 ms)Writing region close event to WAL at 1732490013370Running coprocessor post-close hooks at 1732490013375 (+5 ms)Closed at 1732490013375 2024-11-24T23:13:33,375 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732490008065.eb705a3a04d42fd23798b48207c112de. 2024-11-24T23:13:33,376 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a5362f4dec21f673244ee873440d1779, disabling compactions & flushes 2024-11-24T23:13:33,376 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:33,376 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:33,376 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. after waiting 0 ms 2024-11-24T23:13:33,376 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:33,381 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/data/default/test/a5362f4dec21f673244ee873440d1779/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T23:13:33,382 INFO [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:33,382 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a5362f4dec21f673244ee873440d1779: Waiting for close lock at 1732490013376Running coprocessor pre-close hooks at 1732490013376Disabling compacts and flushes for region at 1732490013376Disabling writes for close at 1732490013376Writing region close event to WAL at 1732490013376Running coprocessor post-close hooks at 1732490013382 (+6 ms)Closed at 1732490013382 2024-11-24T23:13:33,382 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732490008065.a5362f4dec21f673244ee873440d1779. 2024-11-24T23:13:33,471 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,45393,1732490005487; all regions closed. 2024-11-24T23:13:33,472 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,37739,1732490005584; all regions closed. 2024-11-24T23:13:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741865_1041 (size=4204) 2024-11-24T23:13:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741834_1010 (size=7064) 2024-11-24T23:13:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741865_1041 (size=4204) 2024-11-24T23:13:33,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741834_1010 (size=7064) 2024-11-24T23:13:33,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741865_1041 (size=4204) 2024-11-24T23:13:33,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741834_1010 (size=7064) 2024-11-24T23:13:33,479 DEBUG [RS:0;ba7f32532130:45393 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs 2024-11-24T23:13:33,479 INFO [RS:0;ba7f32532130:45393 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba7f32532130%2C45393%2C1732490005487:(num 1732490007246) 2024-11-24T23:13:33,479 DEBUG [RS:0;ba7f32532130:45393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:33,479 INFO [RS:0;ba7f32532130:45393 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T23:13:33,479 DEBUG [RS:1;ba7f32532130:37739 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs 2024-11-24T23:13:33,479 INFO [RS:1;ba7f32532130:37739 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba7f32532130%2C37739%2C1732490005584.meta:.meta(num 1732490011266) 2024-11-24T23:13:33,479 INFO [RS:0;ba7f32532130:45393 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T23:13:33,480 INFO [RS:0;ba7f32532130:45393 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T23:13:33,480 INFO [RS:0;ba7f32532130:45393 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T23:13:33,480 INFO [RS:0;ba7f32532130:45393 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T23:13:33,480 INFO [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T23:13:33,480 INFO [RS:0;ba7f32532130:45393 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T23:13:33,480 INFO [RS:0;ba7f32532130:45393 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T23:13:33,480 INFO [RS:0;ba7f32532130:45393 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45393 2024-11-24T23:13:33,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741835_1011 (size=7074) 2024-11-24T23:13:33,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741835_1011 (size=7074) 2024-11-24T23:13:33,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741835_1011 (size=7074) 2024-11-24T23:13:33,485 DEBUG [RS:1;ba7f32532130:37739 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/oldWALs 2024-11-24T23:13:33,485 INFO [RS:1;ba7f32532130:37739 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ba7f32532130%2C37739%2C1732490005584:(num 1732490007271) 2024-11-24T23:13:33,485 DEBUG [RS:1;ba7f32532130:37739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T23:13:33,485 INFO [RS:1;ba7f32532130:37739 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T23:13:33,485 INFO [RS:1;ba7f32532130:37739 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T23:13:33,486 INFO [RS:1;ba7f32532130:37739 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T23:13:33,486 INFO [RS:1;ba7f32532130:37739 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T23:13:33,486 INFO [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T23:13:33,486 INFO [RS:1;ba7f32532130:37739 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37739 2024-11-24T23:13:33,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,45393,1732490005487 2024-11-24T23:13:33,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T23:13:33,493 INFO [RS:0;ba7f32532130:45393 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T23:13:33,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,37739,1732490005584 2024-11-24T23:13:33,503 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,37739,1732490005584] 2024-11-24T23:13:33,503 INFO [RS:1;ba7f32532130:37739 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T23:13:33,524 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,37739,1732490005584 already deleted, retry=false 2024-11-24T23:13:33,524 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,37739,1732490005584 expired; onlineServers=1 2024-11-24T23:13:33,525 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,45393,1732490005487] 2024-11-24T23:13:33,535 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,45393,1732490005487 already deleted, retry=false 2024-11-24T23:13:33,535 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,45393,1732490005487 expired; onlineServers=0 2024-11-24T23:13:33,535 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba7f32532130,34129,1732490004717' ***** 2024-11-24T23:13:33,535 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T23:13:33,535 INFO [M:0;ba7f32532130:34129 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T23:13:33,535 INFO [M:0;ba7f32532130:34129 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T23:13:33,535 DEBUG [M:0;ba7f32532130:34129 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T23:13:33,536 DEBUG [M:0;ba7f32532130:34129 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T23:13:33,536 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T23:13:33,536 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732490006794 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732490006794,5,FailOnTimeoutGroup] 2024-11-24T23:13:33,536 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732490006794 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732490006794,5,FailOnTimeoutGroup] 2024-11-24T23:13:33,536 INFO [M:0;ba7f32532130:34129 {}] hbase.ChoreService(370): Chore service for: master/ba7f32532130:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T23:13:33,536 INFO [M:0;ba7f32532130:34129 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T23:13:33,536 DEBUG [M:0;ba7f32532130:34129 {}] master.HMaster(1795): Stopping service threads 2024-11-24T23:13:33,536 INFO [M:0;ba7f32532130:34129 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T23:13:33,536 INFO [M:0;ba7f32532130:34129 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T23:13:33,536 INFO [M:0;ba7f32532130:34129 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T23:13:33,537 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T23:13:33,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T23:13:33,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T23:13:33,546 DEBUG [M:0;ba7f32532130:34129 {}] zookeeper.ZKUtil(347): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T23:13:33,546 WARN [M:0;ba7f32532130:34129 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T23:13:33,547 INFO [M:0;ba7f32532130:34129 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/.lastflushedseqids 2024-11-24T23:13:33,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741870_1046 (size=1) 2024-11-24T23:13:33,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741870_1046 (size=1) 2024-11-24T23:13:33,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741870_1046 (size=1) 2024-11-24T23:13:33,562 INFO [M:0;ba7f32532130:34129 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T23:13:33,563 INFO [M:0;ba7f32532130:34129 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T23:13:33,563 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T23:13:33,563 INFO [M:0;ba7f32532130:34129 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T23:13:33,563 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T23:13:33,563 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T23:13:33,563 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T23:13:33,563 INFO [M:0;ba7f32532130:34129 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=250.11 KB heapSize=303.94 KB 2024-11-24T23:13:33,581 DEBUG [M:0;ba7f32532130:34129 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/be160d7def564097bdc7de05e15a7fc1 is 82, key is hbase:meta,,1/info:regioninfo/1732490011588/Put/seqid=0 2024-11-24T23:13:33,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741871_1047 (size=6071) 2024-11-24T23:13:33,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741871_1047 (size=6071) 2024-11-24T23:13:33,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741871_1047 (size=6071) 2024-11-24T23:13:33,588 INFO [M:0;ba7f32532130:34129 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/be160d7def564097bdc7de05e15a7fc1 2024-11-24T23:13:33,612 DEBUG [M:0;ba7f32532130:34129 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8da7a070a4274d0a92acb9a4afe8e66c is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732490013233/Put/seqid=0 2024-11-24T23:13:33,614 INFO [RS:0;ba7f32532130:45393 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T23:13:33,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T23:13:33,614 INFO [RS:0;ba7f32532130:45393 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,45393,1732490005487; zookeeper connection closed. 2024-11-24T23:13:33,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45393-0x1016f1f81790001, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T23:13:33,614 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@78cca935 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@78cca935 2024-11-24T23:13:33,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741872_1048 (size=12801) 2024-11-24T23:13:33,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741872_1048 (size=12801) 2024-11-24T23:13:33,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741872_1048 (size=12801) 2024-11-24T23:13:33,621 INFO [M:0;ba7f32532130:34129 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.83 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8da7a070a4274d0a92acb9a4afe8e66c 2024-11-24T23:13:33,625 INFO [RS:1;ba7f32532130:37739 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T23:13:33,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T23:13:33,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37739-0x1016f1f81790002, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T23:13:33,625 INFO [RS:1;ba7f32532130:37739 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,37739,1732490005584; zookeeper connection closed. 2024-11-24T23:13:33,625 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54713b2c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54713b2c 2024-11-24T23:13:33,625 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T23:13:33,643 DEBUG [M:0;ba7f32532130:34129 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a8a47e217c64f1f8f3053a4d68b9e84 is 69, key is ba7f32532130,37739,1732490005584/rs:state/1732490006908/Put/seqid=0 2024-11-24T23:13:33,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741873_1049 (size=5442) 2024-11-24T23:13:33,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741873_1049 (size=5442) 2024-11-24T23:13:33,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741873_1049 (size=5442) 2024-11-24T23:13:33,652 INFO [M:0;ba7f32532130:34129 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a8a47e217c64f1f8f3053a4d68b9e84 2024-11-24T23:13:33,658 INFO [M:0;ba7f32532130:34129 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a8a47e217c64f1f8f3053a4d68b9e84 2024-11-24T23:13:33,672 DEBUG [M:0;ba7f32532130:34129 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6966e1c1e4694ce2889c1174dae8da26 is 52, key is load_balancer_on/state:d/1732490010382/Put/seqid=0 2024-11-24T23:13:33,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741874_1050 (size=5058) 2024-11-24T23:13:33,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741874_1050 (size=5058) 2024-11-24T23:13:33,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741874_1050 (size=5058) 2024-11-24T23:13:33,680 INFO [M:0;ba7f32532130:34129 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6966e1c1e4694ce2889c1174dae8da26 2024-11-24T23:13:33,688 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/be160d7def564097bdc7de05e15a7fc1 as hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/be160d7def564097bdc7de05e15a7fc1 2024-11-24T23:13:33,694 INFO [M:0;ba7f32532130:34129 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/be160d7def564097bdc7de05e15a7fc1, entries=14, sequenceid=575, filesize=5.9 K 2024-11-24T23:13:33,695 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8da7a070a4274d0a92acb9a4afe8e66c as hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8da7a070a4274d0a92acb9a4afe8e66c 2024-11-24T23:13:33,702 INFO [M:0;ba7f32532130:34129 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8da7a070a4274d0a92acb9a4afe8e66c, entries=79, sequenceid=575, filesize=12.5 K 2024-11-24T23:13:33,703 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a8a47e217c64f1f8f3053a4d68b9e84 as hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a8a47e217c64f1f8f3053a4d68b9e84 2024-11-24T23:13:33,710 INFO [M:0;ba7f32532130:34129 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a8a47e217c64f1f8f3053a4d68b9e84 2024-11-24T23:13:33,710 INFO [M:0;ba7f32532130:34129 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a8a47e217c64f1f8f3053a4d68b9e84, entries=3, sequenceid=575, filesize=5.3 K 2024-11-24T23:13:33,712 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6966e1c1e4694ce2889c1174dae8da26 as hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6966e1c1e4694ce2889c1174dae8da26 2024-11-24T23:13:33,718 INFO [M:0;ba7f32532130:34129 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41705/user/jenkins/test-data/b7dfa3b3-36aa-2431-468a-a4163ba3d4f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6966e1c1e4694ce2889c1174dae8da26, entries=1, sequenceid=575, filesize=4.9 K 2024-11-24T23:13:33,719 INFO [M:0;ba7f32532130:34129 {}] regionserver.HRegion(3140): Finished flush of dataSize ~250.11 KB/256108, heapSize ~303.88 KB/311168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=575, compaction requested=false 2024-11-24T23:13:33,721 INFO [M:0;ba7f32532130:34129 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T23:13:33,721 DEBUG [M:0;ba7f32532130:34129 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732490013563Disabling compacts and flushes for region at 1732490013563Disabling writes for close at 1732490013563Obtaining lock to block concurrent updates at 1732490013563Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732490013563Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=256108, getHeapSize=311168, getOffHeapSize=0, getCellsCount=714 at 1732490013564 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732490013564Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732490013564Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732490013580 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732490013580Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732490013594 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732490013611 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732490013611Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732490013627 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732490013643 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732490013643Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732490013658 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732490013672 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732490013672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7029a4be: reopening flushed file at 1732490013686 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61eba6e9: reopening flushed file at 1732490013694 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9c5bf5c: reopening flushed file at 1732490013702 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e144fdc: reopening flushed file at 1732490013710 (+8 ms)Finished flush of dataSize ~250.11 KB/256108, heapSize ~303.88 KB/311168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=575, compaction requested=false at 1732490013719 (+9 ms)Writing region close event to WAL at 1732490013721 (+2 ms)Closed at 1732490013721 2024-11-24T23:13:33,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40001 is added to blk_1073741830_1006 (size=294492) 2024-11-24T23:13:33,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39131 is added to blk_1073741830_1006 (size=294492) 2024-11-24T23:13:33,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44853 is added to blk_1073741830_1006 (size=294492) 2024-11-24T23:13:33,727 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T23:13:33,727 INFO [M:0;ba7f32532130:34129 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T23:13:33,727 INFO [M:0;ba7f32532130:34129 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34129 2024-11-24T23:13:33,727 INFO [M:0;ba7f32532130:34129 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T23:13:33,835 INFO [M:0;ba7f32532130:34129 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T23:13:33,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T23:13:33,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34129-0x1016f1f81790000, quorum=127.0.0.1:60106, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T23:13:33,840 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T23:13:33,843 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T23:13:33,843 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T23:13:33,843 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T23:13:33,843 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.log.dir/,STOPPED} 2024-11-24T23:13:33,846 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T23:13:33,846 WARN [BP-1799454338-172.17.0.3-1732489999444 heartbeating to localhost/127.0.0.1:41705 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T23:13:33,846 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T23:13:33,846 WARN [BP-1799454338-172.17.0.3-1732489999444 heartbeating to localhost/127.0.0.1:41705 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1799454338-172.17.0.3-1732489999444 (Datanode Uuid 6c3b17cf-6945-4e2b-804c-c4d98c9b57ef) service to localhost/127.0.0.1:41705 2024-11-24T23:13:33,848 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data5/current/BP-1799454338-172.17.0.3-1732489999444 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T23:13:33,848 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data6/current/BP-1799454338-172.17.0.3-1732489999444 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T23:13:33,849 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T23:13:33,852 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T23:13:33,852 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T23:13:33,852 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T23:13:33,852 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T23:13:33,853 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.log.dir/,STOPPED} 2024-11-24T23:13:33,854 WARN [BP-1799454338-172.17.0.3-1732489999444 heartbeating to localhost/127.0.0.1:41705 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T23:13:33,854 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T23:13:33,854 WARN [BP-1799454338-172.17.0.3-1732489999444 heartbeating to localhost/127.0.0.1:41705 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1799454338-172.17.0.3-1732489999444 (Datanode Uuid 2245d66d-029d-4ff6-9903-409dd1be8822) service to localhost/127.0.0.1:41705 2024-11-24T23:13:33,854 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T23:13:33,855 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data3/current/BP-1799454338-172.17.0.3-1732489999444 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T23:13:33,855 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data4/current/BP-1799454338-172.17.0.3-1732489999444 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T23:13:33,855 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T23:13:33,857 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T23:13:33,858 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T23:13:33,858 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T23:13:33,858 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T23:13:33,858 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.log.dir/,STOPPED} 2024-11-24T23:13:33,859 WARN [BP-1799454338-172.17.0.3-1732489999444 heartbeating to localhost/127.0.0.1:41705 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T23:13:33,859 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T23:13:33,859 WARN [BP-1799454338-172.17.0.3-1732489999444 heartbeating to localhost/127.0.0.1:41705 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1799454338-172.17.0.3-1732489999444 (Datanode Uuid 4c22eff2-9f74-488e-8ab8-03f0164c7ebb) service to localhost/127.0.0.1:41705 2024-11-24T23:13:33,859 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T23:13:33,860 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data1/current/BP-1799454338-172.17.0.3-1732489999444 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T23:13:33,861 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/cluster_d0129fca-bd47-0cbc-e6e7-3c1b5256c800/data/data2/current/BP-1799454338-172.17.0.3-1732489999444 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T23:13:33,861 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T23:13:33,869 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T23:13:33,870 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T23:13:33,870 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T23:13:33,870 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T23:13:33,871 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a09af5c-e614-cc24-7d6e-84d2f8d453f9/hadoop.log.dir/,STOPPED} 2024-11-24T23:13:33,882 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T23:13:33,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down