2024-11-28 07:01:46,936 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-28 07:01:46,957 main DEBUG Took 0.018991 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 07:01:46,958 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 07:01:46,958 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 07:01:46,960 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 07:01:46,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,971 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 07:01:46,983 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,985 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,986 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,986 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,986 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,987 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,988 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,989 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,990 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,991 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,991 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,992 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,993 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,994 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,994 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,995 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:01:46,995 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,995 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 07:01:46,997 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:01:46,998 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 07:01:46,999 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 07:01:47,000 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 07:01:47,001 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 07:01:47,001 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 07:01:47,009 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 07:01:47,012 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 07:01:47,013 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 07:01:47,014 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 07:01:47,014 main DEBUG createAppenders(={Console}) 2024-11-28 07:01:47,015 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-28 07:01:47,015 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-28 07:01:47,015 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-28 07:01:47,016 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 07:01:47,016 main DEBUG OutputStream closed 2024-11-28 07:01:47,017 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 07:01:47,017 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 07:01:47,017 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-28 07:01:47,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 07:01:47,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 07:01:47,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 07:01:47,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 07:01:47,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 07:01:47,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 07:01:47,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 07:01:47,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 07:01:47,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 07:01:47,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 07:01:47,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 07:01:47,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 07:01:47,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 07:01:47,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 07:01:47,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 07:01:47,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 07:01:47,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 07:01:47,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 07:01:47,098 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 07:01:47,098 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-28 07:01:47,098 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 07:01:47,099 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-28T07:01:47,111 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-28 07:01:47,114 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 07:01:47,114 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T07:01:47,323 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1 2024-11-28T07:01:47,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T07:01:47,361 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b, deleteOnExit=true 2024-11-28T07:01:47,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T07:01:47,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/test.cache.data in system properties and HBase conf 2024-11-28T07:01:47,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T07:01:47,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/hadoop.log.dir in system properties and HBase conf 2024-11-28T07:01:47,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T07:01:47,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T07:01:47,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T07:01:47,458 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T07:01:47,554 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:01:47,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T07:01:47,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T07:01:47,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T07:01:47,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T07:01:47,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T07:01:47,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T07:01:47,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T07:01:47,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T07:01:47,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T07:01:47,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/nfs.dump.dir in system properties and HBase conf 2024-11-28T07:01:47,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/java.io.tmpdir in system properties and HBase conf 2024-11-28T07:01:47,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T07:01:47,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T07:01:47,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T07:01:48,597 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T07:01:48,663 INFO [Time-limited test {}] log.Log(170): Logging initialized @2378ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T07:01:48,723 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T07:01:48,779 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T07:01:48,799 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T07:01:48,799 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T07:01:48,801 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T07:01:48,812 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T07:01:48,814 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/hadoop.log.dir/,AVAILABLE} 2024-11-28T07:01:48,815 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T07:01:48,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/java.io.tmpdir/jetty-localhost-34865-hadoop-hdfs-3_4_1-tests_jar-_-any-17026737675862538760/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T07:01:48,995 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:34865} 2024-11-28T07:01:48,995 INFO [Time-limited test {}] server.Server(415): Started @2712ms 2024-11-28T07:01:49,504 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T07:01:49,511 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T07:01:49,512 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T07:01:49,512 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T07:01:49,513 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T07:01:49,513 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/hadoop.log.dir/,AVAILABLE} 2024-11-28T07:01:49,514 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T07:01:49,615 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/java.io.tmpdir/jetty-localhost-46205-hadoop-hdfs-3_4_1-tests_jar-_-any-6422536427538408643/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T07:01:49,616 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:46205} 2024-11-28T07:01:49,616 INFO [Time-limited test {}] server.Server(415): Started @3332ms 2024-11-28T07:01:49,665 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T07:01:49,777 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T07:01:49,783 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T07:01:49,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T07:01:49,789 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T07:01:49,789 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T07:01:49,790 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/hadoop.log.dir/,AVAILABLE} 2024-11-28T07:01:49,791 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T07:01:49,891 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/java.io.tmpdir/jetty-localhost-37301-hadoop-hdfs-3_4_1-tests_jar-_-any-17523352239131675445/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T07:01:49,892 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:37301} 2024-11-28T07:01:49,892 INFO [Time-limited test {}] server.Server(415): Started @3608ms 2024-11-28T07:01:49,895 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T07:01:49,940 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T07:01:49,946 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T07:01:49,948 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T07:01:49,949 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T07:01:49,949 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T07:01:49,950 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/hadoop.log.dir/,AVAILABLE} 2024-11-28T07:01:49,951 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T07:01:50,052 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/java.io.tmpdir/jetty-localhost-41545-hadoop-hdfs-3_4_1-tests_jar-_-any-8286013619320561390/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T07:01:50,053 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:41545} 2024-11-28T07:01:50,053 INFO [Time-limited test {}] server.Server(415): Started @3769ms 2024-11-28T07:01:50,057 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T07:01:50,916 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b/data/data2/current/BP-68068463-172.17.0.2-1732777308137/current, will proceed with Du for space computation calculation, 2024-11-28T07:01:50,916 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b/data/data4/current/BP-68068463-172.17.0.2-1732777308137/current, will proceed with Du for space computation calculation, 2024-11-28T07:01:50,916 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b/data/data3/current/BP-68068463-172.17.0.2-1732777308137/current, will proceed with Du for space computation calculation, 2024-11-28T07:01:50,916 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b/data/data1/current/BP-68068463-172.17.0.2-1732777308137/current, will proceed with Du for space computation calculation, 2024-11-28T07:01:50,946 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T07:01:50,947 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T07:01:50,948 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b/data/data5/current/BP-68068463-172.17.0.2-1732777308137/current, will proceed with Du for space computation calculation, 2024-11-28T07:01:50,948 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b/data/data6/current/BP-68068463-172.17.0.2-1732777308137/current, will proceed with Du for space computation calculation, 2024-11-28T07:01:50,969 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T07:01:50,996 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x40a32d5434d736e7 with lease ID 0xa33929cc155690ea: Processing first storage report for DS-cd3e3d2e-47a2-4f27-a3cb-ae1857b37466 from datanode DatanodeRegistration(127.0.0.1:38193, datanodeUuid=1bd9e0ba-3316-4a6e-85a2-7d90ad545c64, infoPort=36335, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137) 2024-11-28T07:01:50,997 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x40a32d5434d736e7 with lease ID 0xa33929cc155690ea: from storage DS-cd3e3d2e-47a2-4f27-a3cb-ae1857b37466 node DatanodeRegistration(127.0.0.1:38193, datanodeUuid=1bd9e0ba-3316-4a6e-85a2-7d90ad545c64, infoPort=36335, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-28T07:01:50,997 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f063e335fec46da with lease ID 0xa33929cc155690e8: Processing first storage report for DS-c4fa8ba8-6773-47fb-a54b-615163e7b781 from datanode DatanodeRegistration(127.0.0.1:38317, datanodeUuid=0d34d3af-ee83-416f-abd9-4d36b6444ff3, infoPort=43245, infoSecurePort=0, ipcPort=38785, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137) 2024-11-28T07:01:50,998 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f063e335fec46da with lease ID 0xa33929cc155690e8: from storage DS-c4fa8ba8-6773-47fb-a54b-615163e7b781 node DatanodeRegistration(127.0.0.1:38317, datanodeUuid=0d34d3af-ee83-416f-abd9-4d36b6444ff3, infoPort=43245, infoSecurePort=0, ipcPort=38785, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T07:01:50,998 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xafc2b5a8cdd63b00 with lease ID 0xa33929cc155690e9: Processing first storage report for DS-26f14f1e-6f3e-493a-a227-da6fa5ec324f from datanode DatanodeRegistration(127.0.0.1:33227, datanodeUuid=dcadf18c-45f8-44ec-a7fe-46d48f5b7cbf, infoPort=40643, infoSecurePort=0, ipcPort=39595, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137) 2024-11-28T07:01:50,998 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xafc2b5a8cdd63b00 with lease ID 0xa33929cc155690e9: from storage DS-26f14f1e-6f3e-493a-a227-da6fa5ec324f node DatanodeRegistration(127.0.0.1:33227, datanodeUuid=dcadf18c-45f8-44ec-a7fe-46d48f5b7cbf, infoPort=40643, infoSecurePort=0, ipcPort=39595, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T07:01:50,999 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x40a32d5434d736e7 with lease ID 0xa33929cc155690ea: Processing first storage report for DS-a226f032-61ea-4c39-a587-9661a9238981 from datanode DatanodeRegistration(127.0.0.1:38193, datanodeUuid=1bd9e0ba-3316-4a6e-85a2-7d90ad545c64, infoPort=36335, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137) 2024-11-28T07:01:50,999 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x40a32d5434d736e7 with lease ID 0xa33929cc155690ea: from storage DS-a226f032-61ea-4c39-a587-9661a9238981 node DatanodeRegistration(127.0.0.1:38193, datanodeUuid=1bd9e0ba-3316-4a6e-85a2-7d90ad545c64, infoPort=36335, infoSecurePort=0, ipcPort=34497, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T07:01:50,999 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f063e335fec46da with lease ID 0xa33929cc155690e8: Processing first storage report for DS-7e4308bb-5888-4783-96ef-2f11d540c725 from datanode DatanodeRegistration(127.0.0.1:38317, datanodeUuid=0d34d3af-ee83-416f-abd9-4d36b6444ff3, infoPort=43245, infoSecurePort=0, ipcPort=38785, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137) 2024-11-28T07:01:51,000 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f063e335fec46da with lease ID 0xa33929cc155690e8: from storage DS-7e4308bb-5888-4783-96ef-2f11d540c725 node DatanodeRegistration(127.0.0.1:38317, datanodeUuid=0d34d3af-ee83-416f-abd9-4d36b6444ff3, infoPort=43245, infoSecurePort=0, ipcPort=38785, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T07:01:51,000 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xafc2b5a8cdd63b00 with lease ID 0xa33929cc155690e9: Processing first storage report for DS-99055227-8190-41df-a31f-949373b27d86 from datanode DatanodeRegistration(127.0.0.1:33227, datanodeUuid=dcadf18c-45f8-44ec-a7fe-46d48f5b7cbf, infoPort=40643, infoSecurePort=0, ipcPort=39595, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137) 2024-11-28T07:01:51,000 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xafc2b5a8cdd63b00 with lease ID 0xa33929cc155690e9: from storage DS-99055227-8190-41df-a31f-949373b27d86 node DatanodeRegistration(127.0.0.1:33227, datanodeUuid=dcadf18c-45f8-44ec-a7fe-46d48f5b7cbf, infoPort=40643, infoSecurePort=0, ipcPort=39595, storageInfo=lv=-57;cid=testClusterID;nsid=1114200608;c=1732777308137), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T07:01:51,078 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1 2024-11-28T07:01:51,140 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b/zookeeper_0, clientPort=49831, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6bb100a3-3f7c-3ede-b413-8e1aaf82dcb1/cluster_8291ee4b-a854-d843-8b5e-c7ae1188ec4b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T07:01:51,150 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49831 2024-11-28T07:01:51,165 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:51,169 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:51,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741825_1001 (size=7) 2024-11-28T07:01:51,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741825_1001 (size=7) 2024-11-28T07:01:51,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741825_1001 (size=7) 2024-11-28T07:01:51,776 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 with version=8 2024-11-28T07:01:51,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/hbase-staging 2024-11-28T07:01:51,850 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T07:01:52,090 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T07:01:52,099 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,099 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,104 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:01:52,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:01:52,233 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T07:01:52,293 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T07:01:52,303 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T07:01:52,306 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:01:52,331 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 114807 (auto-detected) 2024-11-28T07:01:52,332 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T07:01:52,351 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40243 2024-11-28T07:01:52,372 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40243 connecting to ZooKeeper ensemble=127.0.0.1:49831 2024-11-28T07:01:52,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:402430x0, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:01:52,476 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40243-0x10181890e5a0000 connected 2024-11-28T07:01:52,572 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:52,579 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:52,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:01:52,596 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929, hbase.cluster.distributed=false 2024-11-28T07:01:52,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:01:52,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40243 2024-11-28T07:01:52,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40243 2024-11-28T07:01:52,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40243 2024-11-28T07:01:52,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40243 2024-11-28T07:01:52,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40243 2024-11-28T07:01:52,720 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T07:01:52,721 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,721 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,722 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:01:52,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,722 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:01:52,724 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T07:01:52,727 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:01:52,728 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42757 2024-11-28T07:01:52,729 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42757 connecting to ZooKeeper ensemble=127.0.0.1:49831 2024-11-28T07:01:52,730 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:52,734 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:52,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427570x0, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:01:52,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:01:52,769 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42757-0x10181890e5a0001 connected 2024-11-28T07:01:52,775 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T07:01:52,784 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T07:01:52,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T07:01:52,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:01:52,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42757 2024-11-28T07:01:52,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42757 2024-11-28T07:01:52,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42757 2024-11-28T07:01:52,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42757 2024-11-28T07:01:52,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42757 2024-11-28T07:01:52,809 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T07:01:52,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,810 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:01:52,810 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,810 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:01:52,810 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T07:01:52,811 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:01:52,812 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38403 2024-11-28T07:01:52,813 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38403 connecting to ZooKeeper ensemble=127.0.0.1:49831 2024-11-28T07:01:52,814 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:52,818 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:52,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384030x0, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:01:52,848 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38403-0x10181890e5a0002 connected 2024-11-28T07:01:52,848 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:01:52,849 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T07:01:52,850 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T07:01:52,851 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T07:01:52,852 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:01:52,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38403 2024-11-28T07:01:52,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38403 2024-11-28T07:01:52,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38403 2024-11-28T07:01:52,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38403 2024-11-28T07:01:52,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38403 2024-11-28T07:01:52,875 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T07:01:52,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,876 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:01:52,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:01:52,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:01:52,876 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T07:01:52,877 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:01:52,877 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44721 2024-11-28T07:01:52,879 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44721 connecting to ZooKeeper ensemble=127.0.0.1:49831 2024-11-28T07:01:52,880 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:52,883 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:52,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447210x0, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:01:52,898 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:447210x0, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:01:52,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44721-0x10181890e5a0003 connected 2024-11-28T07:01:52,899 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T07:01:52,900 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T07:01:52,901 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T07:01:52,903 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:01:52,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44721 2024-11-28T07:01:52,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44721 2024-11-28T07:01:52,907 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44721 2024-11-28T07:01:52,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44721 2024-11-28T07:01:52,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44721 2024-11-28T07:01:52,925 DEBUG [M:0;1cb9cf437a36:40243 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1cb9cf437a36:40243 2024-11-28T07:01:52,926 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:52,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:01:52,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:01:52,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:01:52,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:01:52,946 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:52,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T07:01:52,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T07:01:52,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T07:01:52,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:52,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:52,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:52,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:52,977 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T07:01:52,979 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1cb9cf437a36,40243,1732777311941 from backup master directory 2024-11-28T07:01:52,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:52,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:01:52,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:01:52,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:01:52,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:01:52,990 WARN [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:01:52,990 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:52,992 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T07:01:52,993 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T07:01:53,043 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/hbase.id] with ID: 15a2db50-5199-4be0-a816-ae6d383659bb 2024-11-28T07:01:53,043 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/.tmp/hbase.id 2024-11-28T07:01:53,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741826_1002 (size=42) 2024-11-28T07:01:53,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741826_1002 (size=42) 2024-11-28T07:01:53,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741826_1002 (size=42) 2024-11-28T07:01:53,058 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/.tmp/hbase.id]:[hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/hbase.id] 2024-11-28T07:01:53,104 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:01:53,111 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T07:01:53,129 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-28T07:01:53,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741827_1003 (size=196) 2024-11-28T07:01:53,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741827_1003 (size=196) 2024-11-28T07:01:53,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741827_1003 (size=196) 2024-11-28T07:01:53,198 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:01:53,199 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T07:01:53,212 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:01:53,215 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:01:53,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741828_1004 (size=1189) 2024-11-28T07:01:53,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741828_1004 (size=1189) 2024-11-28T07:01:53,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741828_1004 (size=1189) 2024-11-28T07:01:53,263 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/data/master/store 2024-11-28T07:01:53,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741829_1005 (size=34) 2024-11-28T07:01:53,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741829_1005 (size=34) 2024-11-28T07:01:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741829_1005 (size=34) 2024-11-28T07:01:53,292 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T07:01:53,295 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:53,296 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T07:01:53,297 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:01:53,297 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:01:53,298 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T07:01:53,299 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:01:53,299 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:01:53,300 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732777313296Disabling compacts and flushes for region at 1732777313296Disabling writes for close at 1732777313298 (+2 ms)Writing region close event to WAL at 1732777313299 (+1 ms)Closed at 1732777313299 2024-11-28T07:01:53,302 WARN [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/data/master/store/.initializing 2024-11-28T07:01:53,302 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/WALs/1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:53,312 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:01:53,328 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C40243%2C1732777311941, suffix=, logDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/WALs/1cb9cf437a36,40243,1732777311941, archiveDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/oldWALs, maxLogs=10 2024-11-28T07:01:53,350 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/WALs/1cb9cf437a36,40243,1732777311941/1cb9cf437a36%2C40243%2C1732777311941.1732777313332, exclude list is [], retry=0 2024-11-28T07:01:53,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38193,DS-cd3e3d2e-47a2-4f27-a3cb-ae1857b37466,DISK] 2024-11-28T07:01:53,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33227,DS-26f14f1e-6f3e-493a-a227-da6fa5ec324f,DISK] 2024-11-28T07:01:53,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38317,DS-c4fa8ba8-6773-47fb-a54b-615163e7b781,DISK] 2024-11-28T07:01:53,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T07:01:53,414 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/WALs/1cb9cf437a36,40243,1732777311941/1cb9cf437a36%2C40243%2C1732777311941.1732777313332 2024-11-28T07:01:53,415 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43245:43245),(127.0.0.1/127.0.0.1:36335:36335),(127.0.0.1/127.0.0.1:40643:40643)] 2024-11-28T07:01:53,416 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:01:53,416 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:53,419 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,421 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T07:01:53,483 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:53,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:53,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,490 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T07:01:53,490 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:53,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:53,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T07:01:53,495 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:53,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:53,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,500 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T07:01:53,500 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:53,502 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:53,502 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,506 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,507 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,513 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,513 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,516 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:01:53,520 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:01:53,526 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:53,527 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74618618, jitterRate=0.11190405488014221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:01:53,534 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732777313432Initializing all the Stores at 1732777313434 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777313435 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777313435Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777313436 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777313436Cleaning up temporary data from old regions at 1732777313513 (+77 ms)Region opened successfully at 1732777313534 (+21 ms) 2024-11-28T07:01:53,535 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T07:01:53,568 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d6ef968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T07:01:53,597 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T07:01:53,606 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:01:53,606 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:01:53,608 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:01:53,610 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T07:01:53,615 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-28T07:01:53,615 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T07:01:53,642 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T07:01:53,650 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T07:01:53,705 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T07:01:53,709 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T07:01:53,711 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T07:01:53,722 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T07:01:53,724 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T07:01:53,727 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T07:01:53,734 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T07:01:53,736 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T07:01:53,747 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T07:01:53,768 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T07:01:53,776 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T07:01:53,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:01:53,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:01:53,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:01:53,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:01:53,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,793 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1cb9cf437a36,40243,1732777311941, sessionid=0x10181890e5a0000, setting cluster-up flag (Was=false) 2024-11-28T07:01:53,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,856 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T07:01:53,861 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:53,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:53,922 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T07:01:53,926 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:53,936 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T07:01:54,002 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:01:54,011 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T07:01:54,014 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer(746): ClusterId : 15a2db50-5199-4be0-a816-ae6d383659bb 2024-11-28T07:01:54,014 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(746): ClusterId : 15a2db50-5199-4be0-a816-ae6d383659bb 2024-11-28T07:01:54,014 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(746): ClusterId : 15a2db50-5199-4be0-a816-ae6d383659bb 2024-11-28T07:01:54,016 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T07:01:54,016 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T07:01:54,016 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T07:01:54,018 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T07:01:54,023 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1cb9cf437a36,40243,1732777311941 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T07:01:54,041 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T07:01:54,041 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T07:01:54,041 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T07:01:54,041 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T07:01:54,041 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T07:01:54,041 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T07:01:54,041 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:01:54,041 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:01:54,041 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:01:54,042 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:01:54,042 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1cb9cf437a36:0, corePoolSize=10, maxPoolSize=10 2024-11-28T07:01:54,042 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,042 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:01:54,042 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,043 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777344043 2024-11-28T07:01:54,044 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T07:01:54,045 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T07:01:54,048 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:01:54,048 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T07:01:54,049 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T07:01:54,049 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T07:01:54,049 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T07:01:54,049 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T07:01:54,050 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,053 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T07:01:54,053 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T07:01:54,053 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T07:01:54,053 DEBUG [RS:1;1cb9cf437a36:38403 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c6d148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T07:01:54,053 DEBUG [RS:2;1cb9cf437a36:44721 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d17b984, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T07:01:54,053 DEBUG [RS:0;1cb9cf437a36:42757 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c1da6fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T07:01:54,054 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T07:01:54,055 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T07:01:54,055 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T07:01:54,055 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:54,056 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T07:01:54,058 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T07:01:54,058 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T07:01:54,063 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732777314060,5,FailOnTimeoutGroup] 2024-11-28T07:01:54,066 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732777314063,5,FailOnTimeoutGroup] 2024-11-28T07:01:54,066 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,066 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T07:01:54,067 DEBUG [RS:2;1cb9cf437a36:44721 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1cb9cf437a36:44721 2024-11-28T07:01:54,067 DEBUG [RS:1;1cb9cf437a36:38403 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1cb9cf437a36:38403 2024-11-28T07:01:54,067 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,068 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,070 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T07:01:54,070 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T07:01:54,070 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T07:01:54,070 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T07:01:54,070 DEBUG [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T07:01:54,070 DEBUG [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T07:01:54,072 DEBUG [RS:0;1cb9cf437a36:42757 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1cb9cf437a36:42757 2024-11-28T07:01:54,072 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T07:01:54,072 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T07:01:54,072 DEBUG [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T07:01:54,073 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,40243,1732777311941 with port=44721, startcode=1732777312875 2024-11-28T07:01:54,073 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,40243,1732777311941 with port=38403, startcode=1732777312808 2024-11-28T07:01:54,073 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,40243,1732777311941 with port=42757, startcode=1732777312688 2024-11-28T07:01:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741831_1007 (size=1321) 2024-11-28T07:01:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741831_1007 (size=1321) 2024-11-28T07:01:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741831_1007 (size=1321) 2024-11-28T07:01:54,081 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T07:01:54,081 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:54,085 DEBUG [RS:2;1cb9cf437a36:44721 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T07:01:54,085 DEBUG [RS:0;1cb9cf437a36:42757 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T07:01:54,085 DEBUG [RS:1;1cb9cf437a36:38403 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T07:01:54,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741832_1008 (size=32) 2024-11-28T07:01:54,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741832_1008 (size=32) 2024-11-28T07:01:54,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741832_1008 (size=32) 2024-11-28T07:01:54,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:54,101 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T07:01:54,104 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:01:54,104 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:54,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:54,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T07:01:54,108 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:01:54,110 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:54,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:54,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T07:01:54,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:01:54,115 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:54,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:54,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T07:01:54,119 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:01:54,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:54,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:54,121 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:01:54,122 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740 2024-11-28T07:01:54,123 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740 2024-11-28T07:01:54,124 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34821, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T07:01:54,124 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T07:01:54,124 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38307, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T07:01:54,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:01:54,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:01:54,127 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:01:54,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:01:54,130 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:54,132 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] master.ServerManager(517): Registering regionserver=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:54,134 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:54,135 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64128906, jitterRate=-0.044404834508895874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:01:54,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732777314098Initializing all the Stores at 1732777314100 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777314100Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777314101 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777314101Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777314101Cleaning up temporary data from old regions at 1732777314126 (+25 ms)Region opened successfully at 1732777314139 (+13 ms) 2024-11-28T07:01:54,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T07:01:54,139 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T07:01:54,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T07:01:54,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T07:01:54,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T07:01:54,141 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T07:01:54,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732777314139Disabling compacts and flushes for region at 1732777314139Disabling writes for close at 1732777314140 (+1 ms)Writing region close event to WAL at 1732777314141 (+1 ms)Closed at 1732777314141 2024-11-28T07:01:54,143 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:54,143 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] master.ServerManager(517): Registering regionserver=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:54,145 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:01:54,145 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T07:01:54,147 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:54,147 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] master.ServerManager(517): Registering regionserver=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:54,147 DEBUG [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:54,148 DEBUG [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36455 2024-11-28T07:01:54,148 DEBUG [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T07:01:54,148 DEBUG [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:54,148 DEBUG [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36455 2024-11-28T07:01:54,148 DEBUG [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T07:01:54,150 DEBUG [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:54,150 DEBUG [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36455 2024-11-28T07:01:54,150 DEBUG [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T07:01:54,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T07:01:54,162 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:01:54,165 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T07:01:54,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:01:54,203 DEBUG [RS:0;1cb9cf437a36:42757 {}] zookeeper.ZKUtil(111): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:54,203 WARN [RS:0;1cb9cf437a36:42757 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:01:54,203 INFO [RS:0;1cb9cf437a36:42757 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:01:54,203 DEBUG [RS:2;1cb9cf437a36:44721 {}] zookeeper.ZKUtil(111): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:54,203 DEBUG [RS:1;1cb9cf437a36:38403 {}] zookeeper.ZKUtil(111): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:54,203 WARN [RS:2;1cb9cf437a36:44721 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:01:54,203 WARN [RS:1;1cb9cf437a36:38403 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:01:54,203 DEBUG [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:54,204 INFO [RS:2;1cb9cf437a36:44721 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:01:54,204 INFO [RS:1;1cb9cf437a36:38403 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:01:54,204 DEBUG [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:54,204 DEBUG [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:54,206 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1cb9cf437a36,38403,1732777312808] 2024-11-28T07:01:54,206 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1cb9cf437a36,42757,1732777312688] 2024-11-28T07:01:54,206 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1cb9cf437a36,44721,1732777312875] 2024-11-28T07:01:54,231 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T07:01:54,231 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T07:01:54,231 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T07:01:54,246 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T07:01:54,246 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T07:01:54,246 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T07:01:54,251 INFO [RS:1;1cb9cf437a36:38403 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T07:01:54,251 INFO [RS:2;1cb9cf437a36:44721 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T07:01:54,251 INFO [RS:0;1cb9cf437a36:42757 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T07:01:54,251 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,251 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,251 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,252 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T07:01:54,252 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T07:01:54,253 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T07:01:54,258 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T07:01:54,258 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T07:01:54,258 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T07:01:54,259 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,259 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,259 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,260 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:01:54,260 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:01:54,260 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,260 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:01:54,261 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:01:54,261 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:01:54,261 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:01:54,261 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:01:54,261 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:01:54,262 DEBUG [RS:0;1cb9cf437a36:42757 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:01:54,262 DEBUG [RS:1;1cb9cf437a36:38403 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:01:54,267 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,38403,1732777312808-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:01:54,267 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,42757,1732777312688-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:01:54,268 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,268 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,44721,1732777312875-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:01:54,287 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T07:01:54,289 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,38403,1732777312808-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,290 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,290 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.Replication(171): 1cb9cf437a36,38403,1732777312808 started 2024-11-28T07:01:54,291 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T07:01:54,292 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,42757,1732777312688-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,292 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,292 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.Replication(171): 1cb9cf437a36,42757,1732777312688 started 2024-11-28T07:01:54,294 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T07:01:54,294 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,44721,1732777312875-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,295 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,295 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.Replication(171): 1cb9cf437a36,44721,1732777312875 started 2024-11-28T07:01:54,308 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,308 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(1482): Serving as 1cb9cf437a36,38403,1732777312808, RpcServer on 1cb9cf437a36/172.17.0.2:38403, sessionid=0x10181890e5a0002 2024-11-28T07:01:54,309 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T07:01:54,309 DEBUG [RS:1;1cb9cf437a36:38403 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:54,309 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,38403,1732777312808' 2024-11-28T07:01:54,309 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T07:01:54,310 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T07:01:54,311 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T07:01:54,311 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T07:01:54,311 DEBUG [RS:1;1cb9cf437a36:38403 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:54,312 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,38403,1732777312808' 2024-11-28T07:01:54,312 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T07:01:54,312 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T07:01:54,313 DEBUG [RS:1;1cb9cf437a36:38403 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T07:01:54,313 INFO [RS:1;1cb9cf437a36:38403 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T07:01:54,313 INFO [RS:1;1cb9cf437a36:38403 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T07:01:54,314 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,314 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer(1482): Serving as 1cb9cf437a36,42757,1732777312688, RpcServer on 1cb9cf437a36/172.17.0.2:42757, sessionid=0x10181890e5a0001 2024-11-28T07:01:54,314 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T07:01:54,314 DEBUG [RS:0;1cb9cf437a36:42757 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:54,315 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,42757,1732777312688' 2024-11-28T07:01:54,315 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T07:01:54,315 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T07:01:54,316 WARN [1cb9cf437a36:40243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T07:01:54,316 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:54,316 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T07:01:54,316 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(1482): Serving as 1cb9cf437a36,44721,1732777312875, RpcServer on 1cb9cf437a36/172.17.0.2:44721, sessionid=0x10181890e5a0003 2024-11-28T07:01:54,316 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T07:01:54,316 DEBUG [RS:0;1cb9cf437a36:42757 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:54,316 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T07:01:54,316 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,42757,1732777312688' 2024-11-28T07:01:54,316 DEBUG [RS:2;1cb9cf437a36:44721 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:54,316 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T07:01:54,316 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,44721,1732777312875' 2024-11-28T07:01:54,316 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T07:01:54,317 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T07:01:54,317 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T07:01:54,318 DEBUG [RS:0;1cb9cf437a36:42757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T07:01:54,318 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T07:01:54,318 INFO [RS:0;1cb9cf437a36:42757 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T07:01:54,318 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T07:01:54,318 INFO [RS:0;1cb9cf437a36:42757 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T07:01:54,318 DEBUG [RS:2;1cb9cf437a36:44721 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:54,318 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,44721,1732777312875' 2024-11-28T07:01:54,318 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T07:01:54,318 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T07:01:54,319 DEBUG [RS:2;1cb9cf437a36:44721 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T07:01:54,319 INFO [RS:2;1cb9cf437a36:44721 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T07:01:54,319 INFO [RS:2;1cb9cf437a36:44721 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T07:01:54,418 INFO [RS:1;1cb9cf437a36:38403 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:01:54,419 INFO [RS:0;1cb9cf437a36:42757 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:01:54,420 INFO [RS:2;1cb9cf437a36:44721 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:01:54,421 INFO [RS:1;1cb9cf437a36:38403 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C38403%2C1732777312808, suffix=, logDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,38403,1732777312808, archiveDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/oldWALs, maxLogs=32 2024-11-28T07:01:54,422 INFO [RS:0;1cb9cf437a36:42757 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C42757%2C1732777312688, suffix=, logDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688, archiveDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/oldWALs, maxLogs=32 2024-11-28T07:01:54,422 INFO [RS:2;1cb9cf437a36:44721 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C44721%2C1732777312875, suffix=, logDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,44721,1732777312875, archiveDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/oldWALs, maxLogs=32 2024-11-28T07:01:54,438 DEBUG [RS:2;1cb9cf437a36:44721 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,44721,1732777312875/1cb9cf437a36%2C44721%2C1732777312875.1732777314425, exclude list is [], retry=0 2024-11-28T07:01:54,438 DEBUG [RS:1;1cb9cf437a36:38403 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,38403,1732777312808/1cb9cf437a36%2C38403%2C1732777312808.1732777314425, exclude list is [], retry=0 2024-11-28T07:01:54,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38193,DS-cd3e3d2e-47a2-4f27-a3cb-ae1857b37466,DISK] 2024-11-28T07:01:54,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38193,DS-cd3e3d2e-47a2-4f27-a3cb-ae1857b37466,DISK] 2024-11-28T07:01:54,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33227,DS-26f14f1e-6f3e-493a-a227-da6fa5ec324f,DISK] 2024-11-28T07:01:54,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33227,DS-26f14f1e-6f3e-493a-a227-da6fa5ec324f,DISK] 2024-11-28T07:01:54,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38317,DS-c4fa8ba8-6773-47fb-a54b-615163e7b781,DISK] 2024-11-28T07:01:54,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38317,DS-c4fa8ba8-6773-47fb-a54b-615163e7b781,DISK] 2024-11-28T07:01:54,463 INFO [RS:2;1cb9cf437a36:44721 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,44721,1732777312875/1cb9cf437a36%2C44721%2C1732777312875.1732777314425 2024-11-28T07:01:54,463 INFO [RS:1;1cb9cf437a36:38403 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,38403,1732777312808/1cb9cf437a36%2C38403%2C1732777312808.1732777314425 2024-11-28T07:01:54,467 DEBUG [RS:2;1cb9cf437a36:44721 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36335:36335),(127.0.0.1/127.0.0.1:40643:40643),(127.0.0.1/127.0.0.1:43245:43245)] 2024-11-28T07:01:54,467 DEBUG [RS:1;1cb9cf437a36:38403 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36335:36335),(127.0.0.1/127.0.0.1:40643:40643),(127.0.0.1/127.0.0.1:43245:43245)] 2024-11-28T07:01:54,469 DEBUG [RS:0;1cb9cf437a36:42757 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688/1cb9cf437a36%2C42757%2C1732777312688.1732777314425, exclude list is [], retry=0 2024-11-28T07:01:54,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38193,DS-cd3e3d2e-47a2-4f27-a3cb-ae1857b37466,DISK] 2024-11-28T07:01:54,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38317,DS-c4fa8ba8-6773-47fb-a54b-615163e7b781,DISK] 2024-11-28T07:01:54,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33227,DS-26f14f1e-6f3e-493a-a227-da6fa5ec324f,DISK] 2024-11-28T07:01:54,478 INFO [RS:0;1cb9cf437a36:42757 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688/1cb9cf437a36%2C42757%2C1732777312688.1732777314425 2024-11-28T07:01:54,479 DEBUG [RS:0;1cb9cf437a36:42757 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43245:43245),(127.0.0.1/127.0.0.1:36335:36335),(127.0.0.1/127.0.0.1:40643:40643)] 2024-11-28T07:01:54,569 DEBUG [1cb9cf437a36:40243 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-28T07:01:54,579 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T07:01:54,585 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T07:01:54,585 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T07:01:54,585 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-28T07:01:54,585 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:01:54,585 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:01:54,585 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-28T07:01:54,585 INFO [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:01:54,585 INFO [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:01:54,585 INFO [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-28T07:01:54,585 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T07:01:54,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:54,599 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,42757,1732777312688, state=OPENING 2024-11-28T07:01:54,638 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T07:01:54,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:54,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:54,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:54,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:01:54,648 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:54,648 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:54,648 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:54,648 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:54,650 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:01:54,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:54,834 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T07:01:54,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41437, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T07:01:54,850 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T07:01:54,850 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:01:54,851 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T07:01:54,855 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C42757%2C1732777312688.meta, suffix=.meta, logDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688, archiveDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/oldWALs, maxLogs=32 2024-11-28T07:01:54,870 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, exclude list is [], retry=0 2024-11-28T07:01:54,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38317,DS-c4fa8ba8-6773-47fb-a54b-615163e7b781,DISK] 2024-11-28T07:01:54,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33227,DS-26f14f1e-6f3e-493a-a227-da6fa5ec324f,DISK] 2024-11-28T07:01:54,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38193,DS-cd3e3d2e-47a2-4f27-a3cb-ae1857b37466,DISK] 2024-11-28T07:01:54,878 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta 2024-11-28T07:01:54,878 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40643:40643),(127.0.0.1/127.0.0.1:43245:43245),(127.0.0.1/127.0.0.1:36335:36335)] 2024-11-28T07:01:54,879 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:01:54,881 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T07:01:54,883 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T07:01:54,888 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T07:01:54,892 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T07:01:54,892 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:54,893 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T07:01:54,893 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T07:01:54,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T07:01:54,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:01:54,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:54,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:54,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T07:01:54,900 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:01:54,900 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:54,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:54,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T07:01:54,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:01:54,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:54,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:54,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T07:01:54,906 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:01:54,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:54,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:54,907 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:01:54,909 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740 2024-11-28T07:01:54,912 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740 2024-11-28T07:01:54,915 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:01:54,915 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:01:54,916 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:01:54,918 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:01:54,920 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74783139, jitterRate=0.11435560882091522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:01:54,920 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T07:01:54,922 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732777314893Writing region info on filesystem at 1732777314893Initializing all the Stores at 1732777314896 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777314896Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777314896Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777314896Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777314896Cleaning up temporary data from old regions at 1732777314915 (+19 ms)Running coprocessor post-open hooks at 1732777314920 (+5 ms)Region opened successfully at 1732777314922 (+2 ms) 2024-11-28T07:01:54,928 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732777314824 2024-11-28T07:01:54,938 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T07:01:54,939 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T07:01:54,941 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:54,943 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,42757,1732777312688, state=OPEN 2024-11-28T07:01:54,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:54,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:54,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:54,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:54,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:54,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:54,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:54,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:54,998 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:55,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T07:01:55,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,42757,1732777312688 in 345 msec 2024-11-28T07:01:55,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T07:01:55,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 859 msec 2024-11-28T07:01:55,019 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:01:55,019 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T07:01:55,038 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:01:55,039 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1] 2024-11-28T07:01:55,059 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:01:55,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42639, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:01:55,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1130 sec 2024-11-28T07:01:55,079 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732777315079, completionTime=-1 2024-11-28T07:01:55,082 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-28T07:01:55,083 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:01:55,108 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-28T07:01:55,108 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777375108 2024-11-28T07:01:55,109 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777435108 2024-11-28T07:01:55,109 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-28T07:01:55,110 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:55,118 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,40243,1732777311941-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:55,118 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,40243,1732777311941-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:55,118 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,40243,1732777311941-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:55,119 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1cb9cf437a36:40243, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:55,120 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:55,120 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:55,127 DEBUG [master/1cb9cf437a36:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T07:01:55,148 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.158sec 2024-11-28T07:01:55,150 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T07:01:55,151 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T07:01:55,170 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T07:01:55,171 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T07:01:55,171 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T07:01:55,172 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,40243,1732777311941-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:01:55,173 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,40243,1732777311941-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T07:01:55,178 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T07:01:55,180 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T07:01:55,180 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,40243,1732777311941-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:01:55,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b48c89d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:01:55,234 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T07:01:55,234 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T07:01:55,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,40243,-1 for getting cluster id 2024-11-28T07:01:55,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:01:55,249 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a2db50-5199-4be0-a816-ae6d383659bb' 2024-11-28T07:01:55,251 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:01:55,251 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a2db50-5199-4be0-a816-ae6d383659bb" 2024-11-28T07:01:55,252 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b0c187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:01:55,252 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,40243,-1] 2024-11-28T07:01:55,254 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:01:55,256 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:01:55,258 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:01:55,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d42a85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:01:55,261 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:01:55,268 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1] 2024-11-28T07:01:55,269 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:01:55,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:01:55,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:55,294 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:01:55,299 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:55,302 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64005489 2024-11-28T07:01:55,303 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:01:55,305 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:01:55,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:01:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:01:55,323 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T07:01:55,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-28T07:01:55,326 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:55,328 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T07:01:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:01:55,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741837_1013 (size=358) 2024-11-28T07:01:55,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741837_1013 (size=358) 2024-11-28T07:01:55,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741837_1013 (size=358) 2024-11-28T07:01:55,346 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86c81a2b73bbd41aa0789229142fc551, NAME => 'test,,1732777315307.86c81a2b73bbd41aa0789229142fc551.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,347 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f5a06150eff3d66689f377c8cac0f8ad, NAME => 'test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,347 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4276658fc73709d30ba28565a9fbbf03, NAME => 'test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741838_1014 (size=42) 2024-11-28T07:01:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741838_1014 (size=42) 2024-11-28T07:01:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741838_1014 (size=42) 2024-11-28T07:01:55,371 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732777315307.86c81a2b73bbd41aa0789229142fc551.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 86c81a2b73bbd41aa0789229142fc551, disabling compactions & flushes 2024-11-28T07:01:55,372 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:01:55,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:01:55,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. after waiting 0 ms 2024-11-28T07:01:55,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:01:55,372 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:01:55,372 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86c81a2b73bbd41aa0789229142fc551: Waiting for close lock at 1732777315372Disabling compacts and flushes for region at 1732777315372Disabling writes for close at 1732777315372Writing region close event to WAL at 1732777315372Closed at 1732777315372 2024-11-28T07:01:55,373 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 32789c56ce6b6f4105a05768e5d8e631, NAME => 'test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741839_1015 (size=45) 2024-11-28T07:01:55,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741839_1015 (size=45) 2024-11-28T07:01:55,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741839_1015 (size=45) 2024-11-28T07:01:55,383 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f5a06150eff3d66689f377c8cac0f8ad, disabling compactions & flushes 2024-11-28T07:01:55,384 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:01:55,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:01:55,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. after waiting 0 ms 2024-11-28T07:01:55,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:01:55,384 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:01:55,384 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f5a06150eff3d66689f377c8cac0f8ad: Waiting for close lock at 1732777315384Disabling compacts and flushes for region at 1732777315384Disabling writes for close at 1732777315384Writing region close event to WAL at 1732777315384Closed at 1732777315384 2024-11-28T07:01:55,385 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 93afe67b3b08d7a0c25e98ea9a0dc636, NAME => 'test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741840_1016 (size=45) 2024-11-28T07:01:55,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741840_1016 (size=45) 2024-11-28T07:01:55,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741840_1016 (size=45) 2024-11-28T07:01:55,400 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,400 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4276658fc73709d30ba28565a9fbbf03, disabling compactions & flushes 2024-11-28T07:01:55,400 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:01:55,400 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:01:55,400 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. after waiting 0 ms 2024-11-28T07:01:55,400 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:01:55,400 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:01:55,400 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4276658fc73709d30ba28565a9fbbf03: Waiting for close lock at 1732777315400Disabling compacts and flushes for region at 1732777315400Disabling writes for close at 1732777315400Writing region close event to WAL at 1732777315400Closed at 1732777315400 2024-11-28T07:01:55,401 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 64c03d436be892d03f4313bba416e19f, NAME => 'test,fff,1732777315307.64c03d436be892d03f4313bba416e19f.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741841_1017 (size=45) 2024-11-28T07:01:55,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741841_1017 (size=45) 2024-11-28T07:01:55,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741841_1017 (size=45) 2024-11-28T07:01:55,427 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,428 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 32789c56ce6b6f4105a05768e5d8e631, disabling compactions & flushes 2024-11-28T07:01:55,428 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:01:55,428 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:01:55,428 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. after waiting 0 ms 2024-11-28T07:01:55,428 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:01:55,428 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:01:55,428 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 32789c56ce6b6f4105a05768e5d8e631: Waiting for close lock at 1732777315428Disabling compacts and flushes for region at 1732777315428Disabling writes for close at 1732777315428Writing region close event to WAL at 1732777315428Closed at 1732777315428 2024-11-28T07:01:55,429 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3d6a3aee8d008ffd9b93917e3b77bf62, NAME => 'test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:01:55,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741842_1018 (size=45) 2024-11-28T07:01:55,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741842_1018 (size=45) 2024-11-28T07:01:55,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741842_1018 (size=45) 2024-11-28T07:01:55,447 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,447 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 93afe67b3b08d7a0c25e98ea9a0dc636, disabling compactions & flushes 2024-11-28T07:01:55,447 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:01:55,447 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:01:55,448 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. after waiting 0 ms 2024-11-28T07:01:55,448 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:01:55,448 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:01:55,448 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 93afe67b3b08d7a0c25e98ea9a0dc636: Waiting for close lock at 1732777315447Disabling compacts and flushes for region at 1732777315447Disabling writes for close at 1732777315448 (+1 ms)Writing region close event to WAL at 1732777315448Closed at 1732777315448 2024-11-28T07:01:55,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741843_1019 (size=45) 2024-11-28T07:01:55,449 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f41522ba5ab99eefcd7eeb6697805184, NAME => 'test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741843_1019 (size=45) 2024-11-28T07:01:55,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741843_1019 (size=45) 2024-11-28T07:01:55,452 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1732777315307.64c03d436be892d03f4313bba416e19f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,453 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 64c03d436be892d03f4313bba416e19f, disabling compactions & flushes 2024-11-28T07:01:55,453 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:01:55,453 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:01:55,453 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. after waiting 0 ms 2024-11-28T07:01:55,453 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:01:55,453 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:01:55,453 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 64c03d436be892d03f4313bba416e19f: Waiting for close lock at 1732777315452Disabling compacts and flushes for region at 1732777315452Disabling writes for close at 1732777315453 (+1 ms)Writing region close event to WAL at 1732777315453Closed at 1732777315453 2024-11-28T07:01:55,454 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 71bd07899358e3652d4466cdb1dd3176, NAME => 'test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741844_1020 (size=45) 2024-11-28T07:01:55,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741844_1020 (size=45) 2024-11-28T07:01:55,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741844_1020 (size=45) 2024-11-28T07:01:55,475 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,475 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3d6a3aee8d008ffd9b93917e3b77bf62, disabling compactions & flushes 2024-11-28T07:01:55,475 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:01:55,475 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:01:55,475 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. after waiting 0 ms 2024-11-28T07:01:55,475 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:01:55,475 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:01:55,475 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3d6a3aee8d008ffd9b93917e3b77bf62: Waiting for close lock at 1732777315475Disabling compacts and flushes for region at 1732777315475Disabling writes for close at 1732777315475Writing region close event to WAL at 1732777315475Closed at 1732777315475 2024-11-28T07:01:55,476 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c86fc11a5bf22fc0b32716e9c44823d3, NAME => 'test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741845_1021 (size=45) 2024-11-28T07:01:55,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741845_1021 (size=45) 2024-11-28T07:01:55,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741845_1021 (size=45) 2024-11-28T07:01:55,485 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,486 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f41522ba5ab99eefcd7eeb6697805184, disabling compactions & flushes 2024-11-28T07:01:55,486 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:01:55,486 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:01:55,486 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. after waiting 0 ms 2024-11-28T07:01:55,486 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:01:55,486 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:01:55,486 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f41522ba5ab99eefcd7eeb6697805184: Waiting for close lock at 1732777315486Disabling compacts and flushes for region at 1732777315486Disabling writes for close at 1732777315486Writing region close event to WAL at 1732777315486Closed at 1732777315486 2024-11-28T07:01:55,487 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fbb63e696a2f079596ba1efcff5d041b, NAME => 'test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741846_1022 (size=45) 2024-11-28T07:01:55,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741846_1022 (size=45) 2024-11-28T07:01:55,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741846_1022 (size=45) 2024-11-28T07:01:55,493 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,493 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 71bd07899358e3652d4466cdb1dd3176, disabling compactions & flushes 2024-11-28T07:01:55,493 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:01:55,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:01:55,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. after waiting 0 ms 2024-11-28T07:01:55,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:01:55,494 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:01:55,494 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 71bd07899358e3652d4466cdb1dd3176: Waiting for close lock at 1732777315493Disabling compacts and flushes for region at 1732777315493Disabling writes for close at 1732777315494 (+1 ms)Writing region close event to WAL at 1732777315494Closed at 1732777315494 2024-11-28T07:01:55,494 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b32dc238678e85db1609526ddf98ddb1, NAME => 'test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741847_1023 (size=45) 2024-11-28T07:01:55,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741847_1023 (size=45) 2024-11-28T07:01:55,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741847_1023 (size=45) 2024-11-28T07:01:55,504 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,504 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c86fc11a5bf22fc0b32716e9c44823d3, disabling compactions & flushes 2024-11-28T07:01:55,504 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:01:55,505 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:01:55,505 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. after waiting 0 ms 2024-11-28T07:01:55,505 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:01:55,505 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:01:55,505 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c86fc11a5bf22fc0b32716e9c44823d3: Waiting for close lock at 1732777315504Disabling compacts and flushes for region at 1732777315504Disabling writes for close at 1732777315505 (+1 ms)Writing region close event to WAL at 1732777315505Closed at 1732777315505 2024-11-28T07:01:55,506 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2857a2dc2b09c2ea3bb5615768b39d67, NAME => 'test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741848_1024 (size=45) 2024-11-28T07:01:55,518 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,518 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fbb63e696a2f079596ba1efcff5d041b, disabling compactions & flushes 2024-11-28T07:01:55,518 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b. 2024-11-28T07:01:55,518 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b. 2024-11-28T07:01:55,518 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b. after waiting 0 ms 2024-11-28T07:01:55,519 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b. 2024-11-28T07:01:55,519 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b. 2024-11-28T07:01:55,519 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fbb63e696a2f079596ba1efcff5d041b: Waiting for close lock at 1732777315518Disabling compacts and flushes for region at 1732777315518Disabling writes for close at 1732777315518Writing region close event to WAL at 1732777315519 (+1 ms)Closed at 1732777315519 2024-11-28T07:01:55,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741848_1024 (size=45) 2024-11-28T07:01:55,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741848_1024 (size=45) 2024-11-28T07:01:55,520 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fa7f0c541fab82c2773800078745e787, NAME => 'test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741849_1025 (size=45) 2024-11-28T07:01:55,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741849_1025 (size=45) 2024-11-28T07:01:55,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741849_1025 (size=45) 2024-11-28T07:01:55,528 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,528 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b32dc238678e85db1609526ddf98ddb1, disabling compactions & flushes 2024-11-28T07:01:55,528 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:01:55,528 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:01:55,528 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. after waiting 0 ms 2024-11-28T07:01:55,528 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:01:55,528 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:01:55,529 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b32dc238678e85db1609526ddf98ddb1: Waiting for close lock at 1732777315528Disabling compacts and flushes for region at 1732777315528Disabling writes for close at 1732777315528Writing region close event to WAL at 1732777315528Closed at 1732777315528 2024-11-28T07:01:55,529 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 89adcb353365cff3d8866fb113c54483, NAME => 'test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741850_1026 (size=45) 2024-11-28T07:01:55,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741850_1026 (size=45) 2024-11-28T07:01:55,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741850_1026 (size=45) 2024-11-28T07:01:55,542 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,542 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2857a2dc2b09c2ea3bb5615768b39d67, disabling compactions & flushes 2024-11-28T07:01:55,542 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:01:55,542 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:01:55,542 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. after waiting 0 ms 2024-11-28T07:01:55,542 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:01:55,542 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:01:55,543 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2857a2dc2b09c2ea3bb5615768b39d67: Waiting for close lock at 1732777315542Disabling compacts and flushes for region at 1732777315542Disabling writes for close at 1732777315542Writing region close event to WAL at 1732777315542Closed at 1732777315542 2024-11-28T07:01:55,543 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6b0e706da5bc6b972222c767b7eb31f6, NAME => 'test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741851_1027 (size=45) 2024-11-28T07:01:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741851_1027 (size=45) 2024-11-28T07:01:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741851_1027 (size=45) 2024-11-28T07:01:55,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fa7f0c541fab82c2773800078745e787, disabling compactions & flushes 2024-11-28T07:01:55,548 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. 2024-11-28T07:01:55,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. 2024-11-28T07:01:55,548 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. after waiting 0 ms 2024-11-28T07:01:55,549 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. 2024-11-28T07:01:55,549 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. 2024-11-28T07:01:55,549 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fa7f0c541fab82c2773800078745e787: Waiting for close lock at 1732777315548Disabling compacts and flushes for region at 1732777315548Disabling writes for close at 1732777315548Writing region close event to WAL at 1732777315549 (+1 ms)Closed at 1732777315549 2024-11-28T07:01:55,549 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fa65e80f1531c2a3d21843d8e252160f, NAME => 'test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741852_1028 (size=45) 2024-11-28T07:01:55,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741852_1028 (size=45) 2024-11-28T07:01:55,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741852_1028 (size=45) 2024-11-28T07:01:55,559 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,559 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 89adcb353365cff3d8866fb113c54483, disabling compactions & flushes 2024-11-28T07:01:55,559 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:01:55,559 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:01:55,559 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. after waiting 0 ms 2024-11-28T07:01:55,559 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:01:55,560 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:01:55,560 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 89adcb353365cff3d8866fb113c54483: Waiting for close lock at 1732777315559Disabling compacts and flushes for region at 1732777315559Disabling writes for close at 1732777315559Writing region close event to WAL at 1732777315560 (+1 ms)Closed at 1732777315560 2024-11-28T07:01:55,561 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1471a7fac83428eb937e98338e4948d6, NAME => 'test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741853_1029 (size=45) 2024-11-28T07:01:55,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741853_1029 (size=45) 2024-11-28T07:01:55,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741853_1029 (size=45) 2024-11-28T07:01:55,569 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,569 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6b0e706da5bc6b972222c767b7eb31f6, disabling compactions & flushes 2024-11-28T07:01:55,569 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:01:55,569 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:01:55,569 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. after waiting 0 ms 2024-11-28T07:01:55,569 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:01:55,569 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:01:55,569 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6b0e706da5bc6b972222c767b7eb31f6: Waiting for close lock at 1732777315569Disabling compacts and flushes for region at 1732777315569Disabling writes for close at 1732777315569Writing region close event to WAL at 1732777315569Closed at 1732777315569 2024-11-28T07:01:55,570 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 46d26e2a6997699edcc98e759a48e66a, NAME => 'test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741854_1030 (size=45) 2024-11-28T07:01:55,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741854_1030 (size=45) 2024-11-28T07:01:55,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741854_1030 (size=45) 2024-11-28T07:01:55,576 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,576 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing fa65e80f1531c2a3d21843d8e252160f, disabling compactions & flushes 2024-11-28T07:01:55,576 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:55,576 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:55,576 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. after waiting 0 ms 2024-11-28T07:01:55,576 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:55,576 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:55,576 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for fa65e80f1531c2a3d21843d8e252160f: Waiting for close lock at 1732777315576Disabling compacts and flushes for region at 1732777315576Disabling writes for close at 1732777315576Writing region close event to WAL at 1732777315576Closed at 1732777315576 2024-11-28T07:01:55,577 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8db2bf432be6aeda43c73169f2d030a4, NAME => 'test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741855_1031 (size=45) 2024-11-28T07:01:55,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741855_1031 (size=45) 2024-11-28T07:01:55,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741855_1031 (size=45) 2024-11-28T07:01:55,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 1471a7fac83428eb937e98338e4948d6, disabling compactions & flushes 2024-11-28T07:01:55,585 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:01:55,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:01:55,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. after waiting 0 ms 2024-11-28T07:01:55,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:01:55,585 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:01:55,585 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1471a7fac83428eb937e98338e4948d6: Waiting for close lock at 1732777315585Disabling compacts and flushes for region at 1732777315585Disabling writes for close at 1732777315585Writing region close event to WAL at 1732777315585Closed at 1732777315585 2024-11-28T07:01:55,586 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 24cb8e2edb958f7d862266464b772c73, NAME => 'test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741856_1032 (size=45) 2024-11-28T07:01:55,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741857_1033 (size=45) 2024-11-28T07:01:55,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741856_1032 (size=45) 2024-11-28T07:01:55,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741857_1033 (size=45) 2024-11-28T07:01:55,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741857_1033 (size=45) 2024-11-28T07:01:55,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741856_1032 (size=45) 2024-11-28T07:01:55,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,601 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 46d26e2a6997699edcc98e759a48e66a, disabling compactions & flushes 2024-11-28T07:01:55,601 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:01:55,601 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8db2bf432be6aeda43c73169f2d030a4, disabling compactions & flushes 2024-11-28T07:01:55,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:01:55,601 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:01:55,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. after waiting 0 ms 2024-11-28T07:01:55,601 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:01:55,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:01:55,602 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. after waiting 0 ms 2024-11-28T07:01:55,602 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:01:55,602 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:01:55,602 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:01:55,602 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 46d26e2a6997699edcc98e759a48e66a: Waiting for close lock at 1732777315601Disabling compacts and flushes for region at 1732777315601Disabling writes for close at 1732777315601Writing region close event to WAL at 1732777315602 (+1 ms)Closed at 1732777315602 2024-11-28T07:01:55,602 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8db2bf432be6aeda43c73169f2d030a4: Waiting for close lock at 1732777315601Disabling compacts and flushes for region at 1732777315601Disabling writes for close at 1732777315602 (+1 ms)Writing region close event to WAL at 1732777315602Closed at 1732777315602 2024-11-28T07:01:55,602 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4d610dd7e8db3ac925480924e91da676, NAME => 'test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,603 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5480f78d30eedd25c01f5f1822c0d5d1, NAME => 'test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741858_1034 (size=45) 2024-11-28T07:01:55,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741858_1034 (size=45) 2024-11-28T07:01:55,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741858_1034 (size=45) 2024-11-28T07:01:55,608 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,608 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 24cb8e2edb958f7d862266464b772c73, disabling compactions & flushes 2024-11-28T07:01:55,608 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:01:55,608 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:01:55,608 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. after waiting 0 ms 2024-11-28T07:01:55,608 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:01:55,608 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:01:55,608 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 24cb8e2edb958f7d862266464b772c73: Waiting for close lock at 1732777315608Disabling compacts and flushes for region at 1732777315608Disabling writes for close at 1732777315608Writing region close event to WAL at 1732777315608Closed at 1732777315608 2024-11-28T07:01:55,609 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b6744df8a48cf582efed97c8fb9152b5, NAME => 'test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741860_1036 (size=45) 2024-11-28T07:01:55,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741859_1035 (size=45) 2024-11-28T07:01:55,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741860_1036 (size=45) 2024-11-28T07:01:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741860_1036 (size=45) 2024-11-28T07:01:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741859_1035 (size=45) 2024-11-28T07:01:55,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741859_1035 (size=45) 2024-11-28T07:01:55,620 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,620 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,620 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4d610dd7e8db3ac925480924e91da676, disabling compactions & flushes 2024-11-28T07:01:55,620 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5480f78d30eedd25c01f5f1822c0d5d1, disabling compactions & flushes 2024-11-28T07:01:55,620 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:01:55,620 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:01:55,620 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:01:55,620 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:01:55,620 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. after waiting 0 ms 2024-11-28T07:01:55,620 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. after waiting 0 ms 2024-11-28T07:01:55,620 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:01:55,621 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:01:55,621 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:01:55,621 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:01:55,621 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5480f78d30eedd25c01f5f1822c0d5d1: Waiting for close lock at 1732777315620Disabling compacts and flushes for region at 1732777315620Disabling writes for close at 1732777315620Writing region close event to WAL at 1732777315621 (+1 ms)Closed at 1732777315621 2024-11-28T07:01:55,621 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4d610dd7e8db3ac925480924e91da676: Waiting for close lock at 1732777315620Disabling compacts and flushes for region at 1732777315620Disabling writes for close at 1732777315620Writing region close event to WAL at 1732777315621 (+1 ms)Closed at 1732777315621 2024-11-28T07:01:55,621 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4c79f2e3a60088e6bfee9be7f58096e9, NAME => 'test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,622 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0519f109d08233033823877d6faba4e1, NAME => 'test,zzz,1732777315307.0519f109d08233033823877d6faba4e1.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929 2024-11-28T07:01:55,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741861_1037 (size=45) 2024-11-28T07:01:55,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741861_1037 (size=45) 2024-11-28T07:01:55,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741861_1037 (size=45) 2024-11-28T07:01:55,635 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,635 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b6744df8a48cf582efed97c8fb9152b5, disabling compactions & flushes 2024-11-28T07:01:55,635 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:01:55,635 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:01:55,635 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. after waiting 0 ms 2024-11-28T07:01:55,635 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:01:55,635 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:01:55,635 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b6744df8a48cf582efed97c8fb9152b5: Waiting for close lock at 1732777315635Disabling compacts and flushes for region at 1732777315635Disabling writes for close at 1732777315635Writing region close event to WAL at 1732777315635Closed at 1732777315635 2024-11-28T07:01:55,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741862_1038 (size=45) 2024-11-28T07:01:55,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741862_1038 (size=45) 2024-11-28T07:01:55,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741862_1038 (size=45) 2024-11-28T07:01:55,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4c79f2e3a60088e6bfee9be7f58096e9, disabling compactions & flushes 2024-11-28T07:01:55,646 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:01:55,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:01:55,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. after waiting 0 ms 2024-11-28T07:01:55,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:01:55,646 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:01:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:01:55,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4c79f2e3a60088e6bfee9be7f58096e9: Waiting for close lock at 1732777315646Disabling compacts and flushes for region at 1732777315646Disabling writes for close at 1732777315646Writing region close event to WAL at 1732777315646Closed at 1732777315646 2024-11-28T07:01:55,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741863_1039 (size=42) 2024-11-28T07:01:55,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741863_1039 (size=42) 2024-11-28T07:01:55,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741863_1039 (size=42) 2024-11-28T07:01:55,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1732777315307.0519f109d08233033823877d6faba4e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:55,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 0519f109d08233033823877d6faba4e1, disabling compactions & flushes 2024-11-28T07:01:55,649 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. 2024-11-28T07:01:55,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. 2024-11-28T07:01:55,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. after waiting 0 ms 2024-11-28T07:01:55,649 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. 2024-11-28T07:01:55,650 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. 2024-11-28T07:01:55,650 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0519f109d08233033823877d6faba4e1: Waiting for close lock at 1732777315649Disabling compacts and flushes for region at 1732777315649Disabling writes for close at 1732777315649Writing region close event to WAL at 1732777315649Closed at 1732777315649 2024-11-28T07:01:55,652 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T07:01:55,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732777315307.86c81a2b73bbd41aa0789229142fc551.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732777315307.64c03d436be892d03f4313bba416e19f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,661 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,661 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,661 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,661 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,661 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,662 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,662 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,662 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,662 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,662 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,662 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,663 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,663 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,663 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,663 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732777315307.0519f109d08233033823877d6faba4e1.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732777315653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777315653"}]},"ts":"1732777315653"} 2024-11-28T07:01:55,710 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-28T07:01:55,713 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T07:01:55,716 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732777315713"}]},"ts":"1732777315713"} 2024-11-28T07:01:55,720 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-28T07:01:55,721 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T07:01:55,741 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T07:01:55,742 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T07:01:55,742 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-28T07:01:55,742 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:01:55,742 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:01:55,742 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-28T07:01:55,742 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:01:55,742 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:01:55,742 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-28T07:01:55,742 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T07:01:55,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=86c81a2b73bbd41aa0789229142fc551, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4276658fc73709d30ba28565a9fbbf03, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=32789c56ce6b6f4105a05768e5d8e631, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64c03d436be892d03f4313bba416e19f, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3d6a3aee8d008ffd9b93917e3b77bf62, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=71bd07899358e3652d4466cdb1dd3176, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c86fc11a5bf22fc0b32716e9c44823d3, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b32dc238678e85db1609526ddf98ddb1, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2857a2dc2b09c2ea3bb5615768b39d67, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89adcb353365cff3d8866fb113c54483, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6b0e706da5bc6b972222c767b7eb31f6, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1471a7fac83428eb937e98338e4948d6, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46d26e2a6997699edcc98e759a48e66a, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=24cb8e2edb958f7d862266464b772c73, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5480f78d30eedd25c01f5f1822c0d5d1, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b6744df8a48cf582efed97c8fb9152b5, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c79f2e3a60088e6bfee9be7f58096e9, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN}] 2024-11-28T07:01:55,754 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:01:55,755 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN; state=OFFLINE, location=1cb9cf437a36,42757,1732777312688; forceNewPlan=false, retain=false 2024-11-28T07:01:55,757 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4c79f2e3a60088e6bfee9be7f58096e9, ASSIGN 2024-11-28T07:01:55,758 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4c79f2e3a60088e6bfee9be7f58096e9, ASSIGN; state=OFFLINE, location=1cb9cf437a36,38403,1732777312808; forceNewPlan=false, retain=false 2024-11-28T07:01:55,759 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b6744df8a48cf582efed97c8fb9152b5, ASSIGN 2024-11-28T07:01:55,761 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b6744df8a48cf582efed97c8fb9152b5, ASSIGN; state=OFFLINE, location=1cb9cf437a36,44721,1732777312875; forceNewPlan=false, retain=false 2024-11-28T07:01:55,762 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5480f78d30eedd25c01f5f1822c0d5d1, ASSIGN 2024-11-28T07:01:55,764 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5480f78d30eedd25c01f5f1822c0d5d1, ASSIGN; state=OFFLINE, location=1cb9cf437a36,38403,1732777312808; forceNewPlan=false, retain=false 2024-11-28T07:01:55,765 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:01:55,767 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN; state=OFFLINE, location=1cb9cf437a36,42757,1732777312688; forceNewPlan=false, retain=false 2024-11-28T07:01:55,771 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=24cb8e2edb958f7d862266464b772c73, ASSIGN 2024-11-28T07:01:55,772 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=24cb8e2edb958f7d862266464b772c73, ASSIGN; state=OFFLINE, location=1cb9cf437a36,44721,1732777312875; forceNewPlan=false, retain=false 2024-11-28T07:01:55,774 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:01:55,775 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN; state=OFFLINE, location=1cb9cf437a36,42757,1732777312688; forceNewPlan=false, retain=false 2024-11-28T07:01:55,777 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46d26e2a6997699edcc98e759a48e66a, ASSIGN 2024-11-28T07:01:55,778 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=46d26e2a6997699edcc98e759a48e66a, ASSIGN; state=OFFLINE, location=1cb9cf437a36,38403,1732777312808; forceNewPlan=false, retain=false 2024-11-28T07:01:55,779 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1471a7fac83428eb937e98338e4948d6, ASSIGN 2024-11-28T07:01:55,781 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1471a7fac83428eb937e98338e4948d6, ASSIGN; state=OFFLINE, location=1cb9cf437a36,44721,1732777312875; forceNewPlan=false, retain=false 2024-11-28T07:01:55,782 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, ASSIGN 2024-11-28T07:01:55,783 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, ASSIGN; state=OFFLINE, location=1cb9cf437a36,42757,1732777312688; forceNewPlan=false, retain=false 2024-11-28T07:01:55,784 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6b0e706da5bc6b972222c767b7eb31f6, ASSIGN 2024-11-28T07:01:55,786 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6b0e706da5bc6b972222c767b7eb31f6, ASSIGN; state=OFFLINE, location=1cb9cf437a36,38403,1732777312808; forceNewPlan=false, retain=false 2024-11-28T07:01:55,788 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=89adcb353365cff3d8866fb113c54483, ASSIGN 2024-11-28T07:01:55,789 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=89adcb353365cff3d8866fb113c54483, ASSIGN; state=OFFLINE, location=1cb9cf437a36,44721,1732777312875; forceNewPlan=false, retain=false 2024-11-28T07:01:55,791 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:01:55,792 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN; state=OFFLINE, location=1cb9cf437a36,42757,1732777312688; forceNewPlan=false, retain=false 2024-11-28T07:01:55,793 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2857a2dc2b09c2ea3bb5615768b39d67, ASSIGN 2024-11-28T07:01:55,795 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2857a2dc2b09c2ea3bb5615768b39d67, ASSIGN; state=OFFLINE, location=1cb9cf437a36,38403,1732777312808; forceNewPlan=false, retain=false 2024-11-28T07:01:55,796 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b32dc238678e85db1609526ddf98ddb1, ASSIGN 2024-11-28T07:01:55,797 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b32dc238678e85db1609526ddf98ddb1, ASSIGN; state=OFFLINE, location=1cb9cf437a36,44721,1732777312875; forceNewPlan=false, retain=false 2024-11-28T07:01:55,798 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:01:55,800 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN; state=OFFLINE, location=1cb9cf437a36,42757,1732777312688; forceNewPlan=false, retain=false 2024-11-28T07:01:55,801 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c86fc11a5bf22fc0b32716e9c44823d3, ASSIGN 2024-11-28T07:01:55,802 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c86fc11a5bf22fc0b32716e9c44823d3, ASSIGN; state=OFFLINE, location=1cb9cf437a36,38403,1732777312808; forceNewPlan=false, retain=false 2024-11-28T07:01:55,803 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=71bd07899358e3652d4466cdb1dd3176, ASSIGN 2024-11-28T07:01:55,805 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=71bd07899358e3652d4466cdb1dd3176, ASSIGN; state=OFFLINE, location=1cb9cf437a36,44721,1732777312875; forceNewPlan=false, retain=false 2024-11-28T07:01:55,806 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:01:55,807 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN; state=OFFLINE, location=1cb9cf437a36,42757,1732777312688; forceNewPlan=false, retain=false 2024-11-28T07:01:55,808 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3d6a3aee8d008ffd9b93917e3b77bf62, ASSIGN 2024-11-28T07:01:55,809 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3d6a3aee8d008ffd9b93917e3b77bf62, ASSIGN; state=OFFLINE, location=1cb9cf437a36,38403,1732777312808; forceNewPlan=false, retain=false 2024-11-28T07:01:55,810 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64c03d436be892d03f4313bba416e19f, ASSIGN 2024-11-28T07:01:55,812 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=64c03d436be892d03f4313bba416e19f, ASSIGN; state=OFFLINE, location=1cb9cf437a36,44721,1732777312875; forceNewPlan=false, retain=false 2024-11-28T07:01:55,813 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:01:55,814 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN; state=OFFLINE, location=1cb9cf437a36,42757,1732777312688; forceNewPlan=false, retain=false 2024-11-28T07:01:55,815 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=32789c56ce6b6f4105a05768e5d8e631, ASSIGN 2024-11-28T07:01:55,817 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=32789c56ce6b6f4105a05768e5d8e631, ASSIGN; state=OFFLINE, location=1cb9cf437a36,38403,1732777312808; forceNewPlan=false, retain=false 2024-11-28T07:01:55,818 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:01:55,819 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN; state=OFFLINE, location=1cb9cf437a36,42757,1732777312688; forceNewPlan=false, retain=false 2024-11-28T07:01:55,821 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4276658fc73709d30ba28565a9fbbf03, ASSIGN 2024-11-28T07:01:55,822 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4276658fc73709d30ba28565a9fbbf03, ASSIGN; state=OFFLINE, location=1cb9cf437a36,44721,1732777312875; forceNewPlan=false, retain=false 2024-11-28T07:01:55,823 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=86c81a2b73bbd41aa0789229142fc551, ASSIGN 2024-11-28T07:01:55,824 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=86c81a2b73bbd41aa0789229142fc551, ASSIGN; state=OFFLINE, location=1cb9cf437a36,38403,1732777312808; forceNewPlan=false, retain=false 2024-11-28T07:01:55,912 INFO [1cb9cf437a36:40243 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-28T07:01:55,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=b6744df8a48cf582efed97c8fb9152b5, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:55,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=24cb8e2edb958f7d862266464b772c73, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:55,917 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1471a7fac83428eb937e98338e4948d6, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:55,918 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=89adcb353365cff3d8866fb113c54483, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:55,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b6744df8a48cf582efed97c8fb9152b5, ASSIGN because future has completed 2024-11-28T07:01:55,919 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=b32dc238678e85db1609526ddf98ddb1, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:55,919 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=71bd07899358e3652d4466cdb1dd3176, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:55,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6744df8a48cf582efed97c8fb9152b5, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:01:55,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=24cb8e2edb958f7d862266464b772c73, ASSIGN because future has completed 2024-11-28T07:01:55,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1471a7fac83428eb937e98338e4948d6, ASSIGN because future has completed 2024-11-28T07:01:55,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=89adcb353365cff3d8866fb113c54483, ASSIGN because future has completed 2024-11-28T07:01:55,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b32dc238678e85db1609526ddf98ddb1, ASSIGN because future has completed 2024-11-28T07:01:55,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure b32dc238678e85db1609526ddf98ddb1, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:01:55,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=71bd07899358e3652d4466cdb1dd3176, ASSIGN because future has completed 2024-11-28T07:01:55,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71bd07899358e3652d4466cdb1dd3176, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:01:55,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89adcb353365cff3d8866fb113c54483, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:01:55,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1471a7fac83428eb937e98338e4948d6, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:01:55,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24cb8e2edb958f7d862266464b772c73, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:01:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:01:55,962 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=64c03d436be892d03f4313bba416e19f, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:55,963 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4276658fc73709d30ba28565a9fbbf03, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:55,963 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=0519f109d08233033823877d6faba4e1, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:55,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:55,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:55,966 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=fa65e80f1531c2a3d21843d8e252160f, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:55,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=64c03d436be892d03f4313bba416e19f, ASSIGN because future has completed 2024-11-28T07:01:55,966 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fa7f0c541fab82c2773800078745e787, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:55,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64c03d436be892d03f4313bba416e19f, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:01:55,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4276658fc73709d30ba28565a9fbbf03, ASSIGN because future has completed 2024-11-28T07:01:55,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN because future has completed 2024-11-28T07:01:55,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN because future has completed 2024-11-28T07:01:55,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN because future has completed 2024-11-28T07:01:55,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:55,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, ASSIGN because future has completed 2024-11-28T07:01:55,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN because future has completed 2024-11-28T07:01:55,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:55,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa65e80f1531c2a3d21843d8e252160f, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:55,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:55,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:56,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4276658fc73709d30ba28565a9fbbf03, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:01:56,007 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,008 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,011 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN because future has completed 2024-11-28T07:01:56,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:56,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN because future has completed 2024-11-28T07:01:56,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN because future has completed 2024-11-28T07:01:56,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN because future has completed 2024-11-28T07:01:56,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4c79f2e3a60088e6bfee9be7f58096e9, ASSIGN because future has completed 2024-11-28T07:01:56,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c79f2e3a60088e6bfee9be7f58096e9, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:56,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:56,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:56,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure f41522ba5ab99eefcd7eeb6697805184, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:56,040 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,041 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=46d26e2a6997699edcc98e759a48e66a, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,042 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,042 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,043 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=32789c56ce6b6f4105a05768e5d8e631, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,045 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86c81a2b73bbd41aa0789229142fc551, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5480f78d30eedd25c01f5f1822c0d5d1, ASSIGN because future has completed 2024-11-28T07:01:56,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5480f78d30eedd25c01f5f1822c0d5d1, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:56,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=46d26e2a6997699edcc98e759a48e66a, ASSIGN because future has completed 2024-11-28T07:01:56,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6b0e706da5bc6b972222c767b7eb31f6, ASSIGN because future has completed 2024-11-28T07:01:56,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2857a2dc2b09c2ea3bb5615768b39d67, ASSIGN because future has completed 2024-11-28T07:01:56,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b0e706da5bc6b972222c767b7eb31f6, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:56,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c86fc11a5bf22fc0b32716e9c44823d3, ASSIGN because future has completed 2024-11-28T07:01:56,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3d6a3aee8d008ffd9b93917e3b77bf62, ASSIGN because future has completed 2024-11-28T07:01:56,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=32789c56ce6b6f4105a05768e5d8e631, ASSIGN because future has completed 2024-11-28T07:01:56,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32789c56ce6b6f4105a05768e5d8e631, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:56,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=86c81a2b73bbd41aa0789229142fc551, ASSIGN because future has completed 2024-11-28T07:01:56,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86c81a2b73bbd41aa0789229142fc551, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:56,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d6a3aee8d008ffd9b93917e3b77bf62, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:56,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure c86fc11a5bf22fc0b32716e9c44823d3, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:56,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2857a2dc2b09c2ea3bb5615768b39d67, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:56,075 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T07:01:56,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33049, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T07:01:56,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46d26e2a6997699edcc98e759a48e66a, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:56,084 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:01:56,084 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 71bd07899358e3652d4466cdb1dd3176, NAME => 'test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-28T07:01:56,085 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,085 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,085 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,085 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,088 INFO [StoreOpener-71bd07899358e3652d4466cdb1dd3176-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,090 INFO [StoreOpener-71bd07899358e3652d4466cdb1dd3176-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 71bd07899358e3652d4466cdb1dd3176 columnFamilyName family 2024-11-28T07:01:56,090 DEBUG [StoreOpener-71bd07899358e3652d4466cdb1dd3176-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,091 INFO [StoreOpener-71bd07899358e3652d4466cdb1dd3176-1 {}] regionserver.HStore(327): Store=71bd07899358e3652d4466cdb1dd3176/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,091 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,092 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,092 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,093 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,093 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,095 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,098 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/71bd07899358e3652d4466cdb1dd3176/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,098 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 71bd07899358e3652d4466cdb1dd3176; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68993325, jitterRate=0.028080657124519348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,098 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:01:56,100 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 71bd07899358e3652d4466cdb1dd3176: Running coprocessor pre-open hook at 1732777316086Writing region info on filesystem at 1732777316086Initializing all the Stores at 1732777316087 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316087Cleaning up temporary data from old regions at 1732777316093 (+6 ms)Running coprocessor post-open hooks at 1732777316098 (+5 ms)Region opened successfully at 1732777316100 (+2 ms) 2024-11-28T07:01:56,102 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176., pid=33, masterSystemTime=1732777316075 2024-11-28T07:01:56,105 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:01:56,105 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:01:56,105 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:01:56,106 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1471a7fac83428eb937e98338e4948d6, NAME => 'test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-28T07:01:56,106 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:56,106 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,106 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,107 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,107 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,110 INFO [StoreOpener-1471a7fac83428eb937e98338e4948d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71bd07899358e3652d4466cdb1dd3176, server=1cb9cf437a36,44721,1732777312875 because future has completed 2024-11-28T07:01:56,112 INFO [StoreOpener-1471a7fac83428eb937e98338e4948d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1471a7fac83428eb937e98338e4948d6 columnFamilyName family 2024-11-28T07:01:56,113 DEBUG [StoreOpener-1471a7fac83428eb937e98338e4948d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,114 INFO [StoreOpener-1471a7fac83428eb937e98338e4948d6-1 {}] regionserver.HStore(327): Store=1471a7fac83428eb937e98338e4948d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,114 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,116 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,117 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,118 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,118 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-28T07:01:56,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 71bd07899358e3652d4466cdb1dd3176, server=1cb9cf437a36,44721,1732777312875 in 177 msec 2024-11-28T07:01:56,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=71bd07899358e3652d4466cdb1dd3176, ASSIGN in 373 msec 2024-11-28T07:01:56,121 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,124 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/1471a7fac83428eb937e98338e4948d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,125 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1471a7fac83428eb937e98338e4948d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62415518, jitterRate=-0.06993630528450012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,125 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:01:56,125 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1471a7fac83428eb937e98338e4948d6: Running coprocessor pre-open hook at 1732777316107Writing region info on filesystem at 1732777316107Initializing all the Stores at 1732777316109 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316109Cleaning up temporary data from old regions at 1732777316118 (+9 ms)Running coprocessor post-open hooks at 1732777316125 (+7 ms)Region opened successfully at 1732777316125 2024-11-28T07:01:56,126 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6., pid=35, masterSystemTime=1732777316075 2024-11-28T07:01:56,129 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:01:56,130 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:01:56,130 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:01:56,130 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => b32dc238678e85db1609526ddf98ddb1, NAME => 'test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-28T07:01:56,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:56,130 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,131 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,131 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,131 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,133 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. 2024-11-28T07:01:56,133 INFO [StoreOpener-b32dc238678e85db1609526ddf98ddb1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,133 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => fa7f0c541fab82c2773800078745e787, NAME => 'test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-28T07:01:56,134 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,134 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,134 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1471a7fac83428eb937e98338e4948d6, server=1cb9cf437a36,44721,1732777312875 because future has completed 2024-11-28T07:01:56,135 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,136 INFO [StoreOpener-b32dc238678e85db1609526ddf98ddb1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b32dc238678e85db1609526ddf98ddb1 columnFamilyName family 2024-11-28T07:01:56,136 DEBUG [StoreOpener-b32dc238678e85db1609526ddf98ddb1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,137 INFO [StoreOpener-b32dc238678e85db1609526ddf98ddb1-1 {}] regionserver.HStore(327): Store=b32dc238678e85db1609526ddf98ddb1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,137 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,137 INFO [StoreOpener-fa7f0c541fab82c2773800078745e787-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,138 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,139 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,140 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,140 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=22 2024-11-28T07:01:56,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 1471a7fac83428eb937e98338e4948d6, server=1cb9cf437a36,44721,1732777312875 in 187 msec 2024-11-28T07:01:56,141 INFO [StoreOpener-fa7f0c541fab82c2773800078745e787-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa7f0c541fab82c2773800078745e787 columnFamilyName family 2024-11-28T07:01:56,141 DEBUG [StoreOpener-fa7f0c541fab82c2773800078745e787-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,142 INFO [StoreOpener-fa7f0c541fab82c2773800078745e787-1 {}] regionserver.HStore(327): Store=fa7f0c541fab82c2773800078745e787/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,142 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,143 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1471a7fac83428eb937e98338e4948d6, ASSIGN in 395 msec 2024-11-28T07:01:56,144 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,144 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,145 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,146 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,147 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/b32dc238678e85db1609526ddf98ddb1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,148 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened b32dc238678e85db1609526ddf98ddb1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64834854, jitterRate=-0.03388538956642151}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,148 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:01:56,148 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for b32dc238678e85db1609526ddf98ddb1: Running coprocessor pre-open hook at 1732777316131Writing region info on filesystem at 1732777316131Initializing all the Stores at 1732777316132 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316132Cleaning up temporary data from old regions at 1732777316140 (+8 ms)Running coprocessor post-open hooks at 1732777316148 (+8 ms)Region opened successfully at 1732777316148 2024-11-28T07:01:56,149 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,150 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1., pid=32, masterSystemTime=1732777316075 2024-11-28T07:01:56,156 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:01:56,156 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:01:56,156 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:56,156 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:01:56,156 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa7f0c541fab82c2773800078745e787/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,156 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => b6744df8a48cf582efed97c8fb9152b5, NAME => 'test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-28T07:01:56,157 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,157 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,157 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,157 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened fa7f0c541fab82c2773800078745e787; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64276970, jitterRate=-0.042198508977890015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,158 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa7f0c541fab82c2773800078745e787 2024-11-28T07:01:56,158 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for fa7f0c541fab82c2773800078745e787: Running coprocessor pre-open hook at 1732777316135Writing region info on filesystem at 1732777316135Initializing all the Stores at 1732777316136 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316136Cleaning up temporary data from old regions at 1732777316146 (+10 ms)Running coprocessor post-open hooks at 1732777316158 (+12 ms)Region opened successfully at 1732777316158 2024-11-28T07:01:56,158 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,159 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787., pid=39, masterSystemTime=1732777316127 2024-11-28T07:01:56,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure b32dc238678e85db1609526ddf98ddb1, server=1cb9cf437a36,44721,1732777312875 because future has completed 2024-11-28T07:01:56,162 INFO [StoreOpener-b6744df8a48cf582efed97c8fb9152b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,163 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. 2024-11-28T07:01:56,163 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. 2024-11-28T07:01:56,163 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. 2024-11-28T07:01:56,163 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 0519f109d08233033823877d6faba4e1, NAME => 'test,zzz,1732777315307.0519f109d08233033823877d6faba4e1.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-28T07:01:56,164 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,164 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,zzz,1732777315307.0519f109d08233033823877d6faba4e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,164 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,164 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,165 INFO [StoreOpener-b6744df8a48cf582efed97c8fb9152b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6744df8a48cf582efed97c8fb9152b5 columnFamilyName family 2024-11-28T07:01:56,165 DEBUG [StoreOpener-b6744df8a48cf582efed97c8fb9152b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,166 INFO [StoreOpener-b6744df8a48cf582efed97c8fb9152b5-1 {}] regionserver.HStore(327): Store=b6744df8a48cf582efed97c8fb9152b5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=16 2024-11-28T07:01:56,167 INFO [StoreOpener-0519f109d08233033823877d6faba4e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure b32dc238678e85db1609526ddf98ddb1, server=1cb9cf437a36,44721,1732777312875 in 233 msec 2024-11-28T07:01:56,166 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,169 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,169 INFO [StoreOpener-0519f109d08233033823877d6faba4e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0519f109d08233033823877d6faba4e1 columnFamilyName family 2024-11-28T07:01:56,169 DEBUG [StoreOpener-0519f109d08233033823877d6faba4e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,170 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b32dc238678e85db1609526ddf98ddb1, ASSIGN in 422 msec 2024-11-28T07:01:56,170 INFO [StoreOpener-0519f109d08233033823877d6faba4e1-1 {}] regionserver.HStore(327): Store=0519f109d08233033823877d6faba4e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,171 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,171 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,172 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,172 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,172 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=fa7f0c541fab82c2773800078745e787, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,173 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T07:01:56,173 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,174 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,174 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,175 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,178 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56321, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T07:01:56,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:56,181 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/b6744df8a48cf582efed97c8fb9152b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,182 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened b6744df8a48cf582efed97c8fb9152b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70547146, jitterRate=0.05123439431190491}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,182 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:01:56,183 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for b6744df8a48cf582efed97c8fb9152b5: Running coprocessor pre-open hook at 1732777316158Writing region info on filesystem at 1732777316158Initializing all the Stores at 1732777316161 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316161Cleaning up temporary data from old regions at 1732777316172 (+11 ms)Running coprocessor post-open hooks at 1732777316182 (+10 ms)Region opened successfully at 1732777316182 2024-11-28T07:01:56,184 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/0519f109d08233033823877d6faba4e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,185 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5., pid=31, masterSystemTime=1732777316075 2024-11-28T07:01:56,185 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 0519f109d08233033823877d6faba4e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60373203, jitterRate=-0.1003691703081131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,185 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0519f109d08233033823877d6faba4e1 2024-11-28T07:01:56,185 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 0519f109d08233033823877d6faba4e1: Running coprocessor pre-open hook at 1732777316165Writing region info on filesystem at 1732777316165Initializing all the Stores at 1732777316166 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316166Cleaning up temporary data from old regions at 1732777316174 (+8 ms)Running coprocessor post-open hooks at 1732777316185 (+11 ms)Region opened successfully at 1732777316185 2024-11-28T07:01:56,187 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732777315307.0519f109d08233033823877d6faba4e1., pid=42, masterSystemTime=1732777316127 2024-11-28T07:01:56,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=18 2024-11-28T07:01:56,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,42757,1732777312688 in 203 msec 2024-11-28T07:01:56,189 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:01:56,189 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:01:56,189 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:01:56,189 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 24cb8e2edb958f7d862266464b772c73, NAME => 'test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-28T07:01:56,190 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,190 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,190 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,190 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:56,190 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. 2024-11-28T07:01:56,191 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. 2024-11-28T07:01:56,191 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:01:56,191 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => f5a06150eff3d66689f377c8cac0f8ad, NAME => 'test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-28T07:01:56,191 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,192 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,192 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,192 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,195 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:01:56,195 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 86c81a2b73bbd41aa0789229142fc551, NAME => 'test,,1732777315307.86c81a2b73bbd41aa0789229142fc551.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-28T07:01:56,196 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,196 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,,1732777315307.86c81a2b73bbd41aa0789229142fc551.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6744df8a48cf582efed97c8fb9152b5, server=1cb9cf437a36,44721,1732777312875 because future has completed 2024-11-28T07:01:56,196 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,196 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,199 INFO [StoreOpener-f5a06150eff3d66689f377c8cac0f8ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,199 INFO [StoreOpener-24cb8e2edb958f7d862266464b772c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,199 INFO [StoreOpener-86c81a2b73bbd41aa0789229142fc551-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,201 INFO [StoreOpener-24cb8e2edb958f7d862266464b772c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24cb8e2edb958f7d862266464b772c73 columnFamilyName family 2024-11-28T07:01:56,201 INFO [StoreOpener-f5a06150eff3d66689f377c8cac0f8ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5a06150eff3d66689f377c8cac0f8ad columnFamilyName family 2024-11-28T07:01:56,202 INFO [StoreOpener-86c81a2b73bbd41aa0789229142fc551-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86c81a2b73bbd41aa0789229142fc551 columnFamilyName family 2024-11-28T07:01:56,202 DEBUG [StoreOpener-24cb8e2edb958f7d862266464b772c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,202 DEBUG [StoreOpener-f5a06150eff3d66689f377c8cac0f8ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,202 DEBUG [StoreOpener-86c81a2b73bbd41aa0789229142fc551-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,202 INFO [StoreOpener-86c81a2b73bbd41aa0789229142fc551-1 {}] regionserver.HStore(327): Store=86c81a2b73bbd41aa0789229142fc551/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,203 INFO [StoreOpener-24cb8e2edb958f7d862266464b772c73-1 {}] regionserver.HStore(327): Store=24cb8e2edb958f7d862266464b772c73/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,203 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-28T07:01:56,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure b6744df8a48cf582efed97c8fb9152b5, server=1cb9cf437a36,44721,1732777312875 in 278 msec 2024-11-28T07:01:56,203 INFO [StoreOpener-f5a06150eff3d66689f377c8cac0f8ad-1 {}] regionserver.HStore(327): Store=f5a06150eff3d66689f377c8cac0f8ad/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,203 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,204 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,205 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,205 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,205 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,206 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,206 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,206 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,207 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,207 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,207 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,207 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b6744df8a48cf582efed97c8fb9152b5, ASSIGN in 459 msec 2024-11-28T07:01:56,207 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,207 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,209 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=0519f109d08233033823877d6faba4e1, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,210 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,210 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,211 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:56,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN in 467 msec 2024-11-28T07:01:56,216 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/f5a06150eff3d66689f377c8cac0f8ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,216 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/86c81a2b73bbd41aa0789229142fc551/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,216 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/24cb8e2edb958f7d862266464b772c73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,217 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened f5a06150eff3d66689f377c8cac0f8ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69138993, jitterRate=0.03025127947330475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,217 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:01:56,217 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 86c81a2b73bbd41aa0789229142fc551; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64582044, jitterRate=-0.037652552127838135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,217 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:01:56,217 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for f5a06150eff3d66689f377c8cac0f8ad: Running coprocessor pre-open hook at 1732777316193Writing region info on filesystem at 1732777316193Initializing all the Stores at 1732777316194 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316194Cleaning up temporary data from old regions at 1732777316207 (+13 ms)Running coprocessor post-open hooks at 1732777316217 (+10 ms)Region opened successfully at 1732777316217 2024-11-28T07:01:56,217 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 86c81a2b73bbd41aa0789229142fc551: Running coprocessor pre-open hook at 1732777316196Writing region info on filesystem at 1732777316196Initializing all the Stores at 1732777316198 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316198Cleaning up temporary data from old regions at 1732777316207 (+9 ms)Running coprocessor post-open hooks at 1732777316217 (+10 ms)Region opened successfully at 1732777316217 2024-11-28T07:01:56,219 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 24cb8e2edb958f7d862266464b772c73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65338839, jitterRate=-0.026375427842140198}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,219 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:01:56,219 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad., pid=46, masterSystemTime=1732777316127 2024-11-28T07:01:56,219 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732777315307.86c81a2b73bbd41aa0789229142fc551., pid=52, masterSystemTime=1732777316172 2024-11-28T07:01:56,219 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 24cb8e2edb958f7d862266464b772c73: Running coprocessor pre-open hook at 1732777316190Writing region info on filesystem at 1732777316190Initializing all the Stores at 1732777316192 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316192Cleaning up temporary data from old regions at 1732777316207 (+15 ms)Running coprocessor post-open hooks at 1732777316219 (+12 ms)Region opened successfully at 1732777316219 2024-11-28T07:01:56,220 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73., pid=36, masterSystemTime=1732777316075 2024-11-28T07:01:56,222 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:01:56,222 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:01:56,222 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:56,222 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => fa65e80f1531c2a3d21843d8e252160f, NAME => 'test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-28T07:01:56,223 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,223 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,223 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,223 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=30 2024-11-28T07:01:56,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,42757,1732777312688 in 221 msec 2024-11-28T07:01:56,224 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:01:56,224 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:01:56,224 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:01:56,225 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 32789c56ce6b6f4105a05768e5d8e631, NAME => 'test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-28T07:01:56,225 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,225 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,225 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,225 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,226 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,226 INFO [StoreOpener-fa65e80f1531c2a3d21843d8e252160f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,226 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:01:56,226 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:01:56,227 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:01:56,227 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 4276658fc73709d30ba28565a9fbbf03, NAME => 'test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-28T07:01:56,227 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,227 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,227 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,227 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,230 INFO [StoreOpener-32789c56ce6b6f4105a05768e5d8e631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86c81a2b73bbd41aa0789229142fc551, server=1cb9cf437a36,38403,1732777312808 because future has completed 2024-11-28T07:01:56,230 INFO [StoreOpener-fa65e80f1531c2a3d21843d8e252160f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa65e80f1531c2a3d21843d8e252160f columnFamilyName family 2024-11-28T07:01:56,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:56,231 DEBUG [StoreOpener-fa65e80f1531c2a3d21843d8e252160f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,231 INFO [StoreOpener-4276658fc73709d30ba28565a9fbbf03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,233 INFO [StoreOpener-32789c56ce6b6f4105a05768e5d8e631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32789c56ce6b6f4105a05768e5d8e631 columnFamilyName family 2024-11-28T07:01:56,233 DEBUG [StoreOpener-32789c56ce6b6f4105a05768e5d8e631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,233 INFO [StoreOpener-4276658fc73709d30ba28565a9fbbf03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4276658fc73709d30ba28565a9fbbf03 columnFamilyName family 2024-11-28T07:01:56,233 INFO [StoreOpener-fa65e80f1531c2a3d21843d8e252160f-1 {}] regionserver.HStore(327): Store=fa65e80f1531c2a3d21843d8e252160f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,233 DEBUG [StoreOpener-4276658fc73709d30ba28565a9fbbf03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,234 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24cb8e2edb958f7d862266464b772c73, server=1cb9cf437a36,44721,1732777312875 because future has completed 2024-11-28T07:01:56,234 INFO [StoreOpener-32789c56ce6b6f4105a05768e5d8e631-1 {}] regionserver.HStore(327): Store=32789c56ce6b6f4105a05768e5d8e631/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,234 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,235 INFO [StoreOpener-4276658fc73709d30ba28565a9fbbf03-1 {}] regionserver.HStore(327): Store=4276658fc73709d30ba28565a9fbbf03/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,235 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,235 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,236 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,236 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,236 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,236 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,236 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,236 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,237 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,237 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,237 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,237 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,237 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,239 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,240 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,240 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,244 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/32789c56ce6b6f4105a05768e5d8e631/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=5 2024-11-28T07:01:56,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 86c81a2b73bbd41aa0789229142fc551, server=1cb9cf437a36,38403,1732777312808 in 174 msec 2024-11-28T07:01:56,244 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4276658fc73709d30ba28565a9fbbf03/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,245 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 4276658fc73709d30ba28565a9fbbf03; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73218613, jitterRate=0.09104235470294952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,245 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 32789c56ce6b6f4105a05768e5d8e631; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64147131, jitterRate=-0.044133260846138}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,245 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:01:56,245 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:01:56,245 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 4276658fc73709d30ba28565a9fbbf03: Running coprocessor pre-open hook at 1732777316227Writing region info on filesystem at 1732777316228 (+1 ms)Initializing all the Stores at 1732777316230 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316230Cleaning up temporary data from old regions at 1732777316237 (+7 ms)Running coprocessor post-open hooks at 1732777316245 (+8 ms)Region opened successfully at 1732777316245 2024-11-28T07:01:56,245 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 32789c56ce6b6f4105a05768e5d8e631: Running coprocessor pre-open hook at 1732777316226Writing region info on filesystem at 1732777316226Initializing all the Stores at 1732777316229 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316229Cleaning up temporary data from old regions at 1732777316237 (+8 ms)Running coprocessor post-open hooks at 1732777316245 (+8 ms)Region opened successfully at 1732777316245 2024-11-28T07:01:56,246 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03., pid=43, masterSystemTime=1732777316075 2024-11-28T07:01:56,246 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631., pid=51, masterSystemTime=1732777316172 2024-11-28T07:01:56,248 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa65e80f1531c2a3d21843d8e252160f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=86c81a2b73bbd41aa0789229142fc551, ASSIGN in 499 msec 2024-11-28T07:01:56,249 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened fa65e80f1531c2a3d21843d8e252160f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73018300, jitterRate=0.08805745840072632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,249 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:56,249 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for fa65e80f1531c2a3d21843d8e252160f: Running coprocessor pre-open hook at 1732777316223Writing region info on filesystem at 1732777316223Initializing all the Stores at 1732777316225 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316226 (+1 ms)Cleaning up temporary data from old regions at 1732777316237 (+11 ms)Running coprocessor post-open hooks at 1732777316249 (+12 ms)Region opened successfully at 1732777316249 2024-11-28T07:01:56,250 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:01:56,250 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:01:56,250 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:01:56,250 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 89adcb353365cff3d8866fb113c54483, NAME => 'test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-28T07:01:56,251 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,251 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,251 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,251 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,252 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f., pid=40, masterSystemTime=1732777316127 2024-11-28T07:01:56,252 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:56,255 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:01:56,255 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:01:56,256 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:01:56,256 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 3d6a3aee8d008ffd9b93917e3b77bf62, NAME => 'test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-28T07:01:56,256 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,256 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,257 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,257 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,257 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1cb9cf437a36,44721,1732777312875, table=test, region=4276658fc73709d30ba28565a9fbbf03. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T07:01:56,257 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1cb9cf437a36,44721,1732777312875, table=test, region=24cb8e2edb958f7d862266464b772c73. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T07:01:56,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4276658fc73709d30ba28565a9fbbf03, server=1cb9cf437a36,44721,1732777312875 because future has completed 2024-11-28T07:01:56,258 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:56,258 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:56,258 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:01:56,258 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 93afe67b3b08d7a0c25e98ea9a0dc636, NAME => 'test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-28T07:01:56,258 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,259 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,259 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,259 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,259 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa65e80f1531c2a3d21843d8e252160f, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:56,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=6 2024-11-28T07:01:56,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 4276658fc73709d30ba28565a9fbbf03, server=1cb9cf437a36,44721,1732777312875 in 261 msec 2024-11-28T07:01:56,267 INFO [StoreOpener-89adcb353365cff3d8866fb113c54483-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,269 INFO [StoreOpener-89adcb353365cff3d8866fb113c54483-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89adcb353365cff3d8866fb113c54483 columnFamilyName family 2024-11-28T07:01:56,269 DEBUG [StoreOpener-89adcb353365cff3d8866fb113c54483-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4276658fc73709d30ba28565a9fbbf03, ASSIGN in 521 msec 2024-11-28T07:01:56,270 INFO [StoreOpener-89adcb353365cff3d8866fb113c54483-1 {}] regionserver.HStore(327): Store=89adcb353365cff3d8866fb113c54483/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,271 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,271 INFO [StoreOpener-3d6a3aee8d008ffd9b93917e3b77bf62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,272 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,272 INFO [StoreOpener-93afe67b3b08d7a0c25e98ea9a0dc636-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,273 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,273 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,273 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,273 INFO [StoreOpener-3d6a3aee8d008ffd9b93917e3b77bf62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d6a3aee8d008ffd9b93917e3b77bf62 columnFamilyName family 2024-11-28T07:01:56,274 DEBUG [StoreOpener-3d6a3aee8d008ffd9b93917e3b77bf62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,274 INFO [StoreOpener-3d6a3aee8d008ffd9b93917e3b77bf62-1 {}] regionserver.HStore(327): Store=3d6a3aee8d008ffd9b93917e3b77bf62/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,275 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,275 INFO [StoreOpener-93afe67b3b08d7a0c25e98ea9a0dc636-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93afe67b3b08d7a0c25e98ea9a0dc636 columnFamilyName family 2024-11-28T07:01:56,275 DEBUG [StoreOpener-93afe67b3b08d7a0c25e98ea9a0dc636-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,276 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,276 INFO [StoreOpener-93afe67b3b08d7a0c25e98ea9a0dc636-1 {}] regionserver.HStore(327): Store=93afe67b3b08d7a0c25e98ea9a0dc636/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,277 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,277 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,277 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,278 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,278 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=21 2024-11-28T07:01:56,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure fa65e80f1531c2a3d21843d8e252160f, server=1cb9cf437a36,42757,1732777312688 in 286 msec 2024-11-28T07:01:56,279 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,280 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,281 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/89adcb353365cff3d8866fb113c54483/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,281 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,281 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,281 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,282 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 89adcb353365cff3d8866fb113c54483; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72906070, jitterRate=0.0863851010799408}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,282 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 89adcb353365cff3d8866fb113c54483 2024-11-28T07:01:56,282 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 89adcb353365cff3d8866fb113c54483: Running coprocessor pre-open hook at 1732777316251Writing region info on filesystem at 1732777316251Initializing all the Stores at 1732777316254 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316254Cleaning up temporary data from old regions at 1732777316273 (+19 ms)Running coprocessor post-open hooks at 1732777316282 (+9 ms)Region opened successfully at 1732777316282 2024-11-28T07:01:56,283 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483., pid=34, masterSystemTime=1732777316075 2024-11-28T07:01:56,285 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,286 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/3d6a3aee8d008ffd9b93917e3b77bf62/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, ASSIGN in 535 msec 2024-11-28T07:01:56,286 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 3d6a3aee8d008ffd9b93917e3b77bf62; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62991799, jitterRate=-0.0613490492105484}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,287 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:01:56,287 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 3d6a3aee8d008ffd9b93917e3b77bf62: Running coprocessor pre-open hook at 1732777316261Writing region info on filesystem at 1732777316261Initializing all the Stores at 1732777316263 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316263Cleaning up temporary data from old regions at 1732777316278 (+15 ms)Running coprocessor post-open hooks at 1732777316287 (+9 ms)Region opened successfully at 1732777316287 2024-11-28T07:01:56,288 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:01:56,289 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:01:56,289 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:01:56,289 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 64c03d436be892d03f4313bba416e19f, NAME => 'test,fff,1732777315307.64c03d436be892d03f4313bba416e19f.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-28T07:01:56,290 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,290 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/93afe67b3b08d7a0c25e98ea9a0dc636/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,290 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,fff,1732777315307.64c03d436be892d03f4313bba416e19f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,290 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,290 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,292 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62., pid=53, masterSystemTime=1732777316172 2024-11-28T07:01:56,292 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 93afe67b3b08d7a0c25e98ea9a0dc636; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60059834, jitterRate=-0.10503873229026794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,292 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:01:56,293 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 93afe67b3b08d7a0c25e98ea9a0dc636: Running coprocessor pre-open hook at 1732777316261Writing region info on filesystem at 1732777316261Initializing all the Stores at 1732777316264 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316264Cleaning up temporary data from old regions at 1732777316281 (+17 ms)Running coprocessor post-open hooks at 1732777316292 (+11 ms)Region opened successfully at 1732777316292 2024-11-28T07:01:56,295 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636., pid=47, masterSystemTime=1732777316127 2024-11-28T07:01:56,295 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:01:56,295 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:01:56,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32789c56ce6b6f4105a05768e5d8e631, server=1cb9cf437a36,38403,1732777312808 because future has completed 2024-11-28T07:01:56,295 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:01:56,296 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 5480f78d30eedd25c01f5f1822c0d5d1, NAME => 'test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-28T07:01:56,296 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,296 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,296 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,296 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,299 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:01:56,299 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:01:56,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,299 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b. 2024-11-28T07:01:56,299 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => fbb63e696a2f079596ba1efcff5d041b, NAME => 'test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-28T07:01:56,300 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,300 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,300 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,300 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d6a3aee8d008ffd9b93917e3b77bf62, server=1cb9cf437a36,38403,1732777312808 because future has completed 2024-11-28T07:01:56,303 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:56,307 INFO [StoreOpener-fbb63e696a2f079596ba1efcff5d041b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,307 INFO [StoreOpener-5480f78d30eedd25c01f5f1822c0d5d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,308 INFO [StoreOpener-64c03d436be892d03f4313bba416e19f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,310 INFO [StoreOpener-5480f78d30eedd25c01f5f1822c0d5d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5480f78d30eedd25c01f5f1822c0d5d1 columnFamilyName family 2024-11-28T07:01:56,310 INFO [StoreOpener-fbb63e696a2f079596ba1efcff5d041b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbb63e696a2f079596ba1efcff5d041b columnFamilyName family 2024-11-28T07:01:56,310 DEBUG [StoreOpener-5480f78d30eedd25c01f5f1822c0d5d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,310 DEBUG [StoreOpener-fbb63e696a2f079596ba1efcff5d041b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,311 INFO [StoreOpener-fbb63e696a2f079596ba1efcff5d041b-1 {}] regionserver.HStore(327): Store=fbb63e696a2f079596ba1efcff5d041b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,311 INFO [StoreOpener-5480f78d30eedd25c01f5f1822c0d5d1-1 {}] regionserver.HStore(327): Store=5480f78d30eedd25c01f5f1822c0d5d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,311 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,311 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,312 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,312 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=11 2024-11-28T07:01:56,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 3d6a3aee8d008ffd9b93917e3b77bf62, server=1cb9cf437a36,38403,1732777312808 in 241 msec 2024-11-28T07:01:56,312 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,312 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,313 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,313 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,313 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,313 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,316 INFO [StoreOpener-64c03d436be892d03f4313bba416e19f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64c03d436be892d03f4313bba416e19f columnFamilyName family 2024-11-28T07:01:56,316 DEBUG [StoreOpener-64c03d436be892d03f4313bba416e19f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,316 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,317 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,317 INFO [StoreOpener-64c03d436be892d03f4313bba416e19f-1 {}] regionserver.HStore(327): Store=64c03d436be892d03f4313bba416e19f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,317 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3d6a3aee8d008ffd9b93917e3b77bf62, ASSIGN in 567 msec 2024-11-28T07:01:56,318 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,319 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,321 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,321 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,322 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/5480f78d30eedd25c01f5f1822c0d5d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,322 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fbb63e696a2f079596ba1efcff5d041b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,323 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 5480f78d30eedd25c01f5f1822c0d5d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68417669, jitterRate=0.019502714276313782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,323 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:01:56,323 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 5480f78d30eedd25c01f5f1822c0d5d1: Running coprocessor pre-open hook at 1732777316297Writing region info on filesystem at 1732777316297Initializing all the Stores at 1732777316298 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316299 (+1 ms)Cleaning up temporary data from old regions at 1732777316313 (+14 ms)Running coprocessor post-open hooks at 1732777316323 (+10 ms)Region opened successfully at 1732777316323 2024-11-28T07:01:56,324 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened fbb63e696a2f079596ba1efcff5d041b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74058528, jitterRate=0.10355806350708008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,324 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbb63e696a2f079596ba1efcff5d041b 2024-11-28T07:01:56,324 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for fbb63e696a2f079596ba1efcff5d041b: Running coprocessor pre-open hook at 1732777316300Writing region info on filesystem at 1732777316300Initializing all the Stores at 1732777316303 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316303Cleaning up temporary data from old regions at 1732777316313 (+10 ms)Running coprocessor post-open hooks at 1732777316324 (+11 ms)Region opened successfully at 1732777316324 2024-11-28T07:01:56,324 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=9 2024-11-28T07:01:56,325 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1., pid=49, masterSystemTime=1732777316172 2024-11-28T07:01:56,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,42757,1732777312688 in 289 msec 2024-11-28T07:01:56,326 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b., pid=44, masterSystemTime=1732777316127 2024-11-28T07:01:56,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN in 580 msec 2024-11-28T07:01:56,328 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:01:56,329 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:01:56,329 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:01:56,329 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 2857a2dc2b09c2ea3bb5615768b39d67, NAME => 'test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-28T07:01:56,330 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,330 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,330 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,330 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,330 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b. 2024-11-28T07:01:56,331 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,kkk,1732777315307.fbb63e696a2f079596ba1efcff5d041b. 2024-11-28T07:01:56,331 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:01:56,331 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d610dd7e8db3ac925480924e91da676, NAME => 'test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-28T07:01:56,331 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,331 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,332 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,332 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,333 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/64c03d436be892d03f4313bba416e19f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,334 INFO [StoreOpener-2857a2dc2b09c2ea3bb5615768b39d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,334 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 64c03d436be892d03f4313bba416e19f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69815165, jitterRate=0.040327027440071106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,335 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64c03d436be892d03f4313bba416e19f 2024-11-28T07:01:56,335 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 64c03d436be892d03f4313bba416e19f: Running coprocessor pre-open hook at 1732777316290Writing region info on filesystem at 1732777316290Initializing all the Stores at 1732777316297 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316297Cleaning up temporary data from old regions at 1732777316321 (+24 ms)Running coprocessor post-open hooks at 1732777316335 (+14 ms)Region opened successfully at 1732777316335 2024-11-28T07:01:56,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5480f78d30eedd25c01f5f1822c0d5d1, server=1cb9cf437a36,38403,1732777312808 because future has completed 2024-11-28T07:01:56,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=fbb63e696a2f079596ba1efcff5d041b, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,336 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732777315307.64c03d436be892d03f4313bba416e19f., pid=37, masterSystemTime=1732777316075 2024-11-28T07:01:56,336 INFO [StoreOpener-2857a2dc2b09c2ea3bb5615768b39d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2857a2dc2b09c2ea3bb5615768b39d67 columnFamilyName family 2024-11-28T07:01:56,336 DEBUG [StoreOpener-2857a2dc2b09c2ea3bb5615768b39d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,337 INFO [StoreOpener-2857a2dc2b09c2ea3bb5615768b39d67-1 {}] regionserver.HStore(327): Store=2857a2dc2b09c2ea3bb5615768b39d67/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,337 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,339 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,339 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:56,340 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,340 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,341 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:01:56,341 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:01:56,343 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=27 2024-11-28T07:01:56,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 5480f78d30eedd25c01f5f1822c0d5d1, server=1cb9cf437a36,38403,1732777312808 in 293 msec 2024-11-28T07:01:56,347 INFO [StoreOpener-4d610dd7e8db3ac925480924e91da676-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,348 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/2857a2dc2b09c2ea3bb5615768b39d67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5480f78d30eedd25c01f5f1822c0d5d1, ASSIGN in 599 msec 2024-11-28T07:01:56,349 INFO [StoreOpener-4d610dd7e8db3ac925480924e91da676-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d610dd7e8db3ac925480924e91da676 columnFamilyName family 2024-11-28T07:01:56,349 DEBUG [StoreOpener-4d610dd7e8db3ac925480924e91da676-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,349 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 2857a2dc2b09c2ea3bb5615768b39d67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66286628, jitterRate=-0.012252271175384521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,349 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:01:56,350 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 2857a2dc2b09c2ea3bb5615768b39d67: Running coprocessor pre-open hook at 1732777316330Writing region info on filesystem at 1732777316330Initializing all the Stores at 1732777316332 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316332Cleaning up temporary data from old regions at 1732777316340 (+8 ms)Running coprocessor post-open hooks at 1732777316349 (+9 ms)Region opened successfully at 1732777316350 (+1 ms) 2024-11-28T07:01:56,350 INFO [StoreOpener-4d610dd7e8db3ac925480924e91da676-1 {}] regionserver.HStore(327): Store=4d610dd7e8db3ac925480924e91da676/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,350 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,351 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=64c03d436be892d03f4313bba416e19f, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:56,351 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67., pid=55, masterSystemTime=1732777316172 2024-11-28T07:01:56,351 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,352 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,353 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,353 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64c03d436be892d03f4313bba416e19f, server=1cb9cf437a36,44721,1732777312875 because future has completed 2024-11-28T07:01:56,355 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:01:56,355 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:01:56,355 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:01:56,355 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b0e706da5bc6b972222c767b7eb31f6, NAME => 'test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-28T07:01:56,356 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,356 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,356 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,356 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,358 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=test, region=32789c56ce6b6f4105a05768e5d8e631. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T07:01:56,359 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,359 INFO [StoreOpener-6b0e706da5bc6b972222c767b7eb31f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=15 2024-11-28T07:01:56,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,42757,1732777312688 in 344 msec 2024-11-28T07:01:56,363 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4d610dd7e8db3ac925480924e91da676/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,364 INFO [StoreOpener-6b0e706da5bc6b972222c767b7eb31f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0e706da5bc6b972222c767b7eb31f6 columnFamilyName family 2024-11-28T07:01:56,364 DEBUG [StoreOpener-6b0e706da5bc6b972222c767b7eb31f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,364 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 4d610dd7e8db3ac925480924e91da676; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73540315, jitterRate=0.0958360880613327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,365 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:01:56,365 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 4d610dd7e8db3ac925480924e91da676: Running coprocessor pre-open hook at 1732777316332Writing region info on filesystem at 1732777316332Initializing all the Stores at 1732777316334 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316334Cleaning up temporary data from old regions at 1732777316353 (+19 ms)Running coprocessor post-open hooks at 1732777316365 (+12 ms)Region opened successfully at 1732777316365 2024-11-28T07:01:56,365 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1cb9cf437a36,44721,1732777312875, table=test, region=64c03d436be892d03f4313bba416e19f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T07:01:56,366 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1cb9cf437a36,44721,1732777312875, table=test, region=24cb8e2edb958f7d862266464b772c73. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T07:01:56,366 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676., pid=38, masterSystemTime=1732777316127 2024-11-28T07:01:56,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN in 616 msec 2024-11-28T07:01:56,370 INFO [StoreOpener-6b0e706da5bc6b972222c767b7eb31f6-1 {}] regionserver.HStore(327): Store=6b0e706da5bc6b972222c767b7eb31f6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,370 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,372 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,372 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,373 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,373 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,373 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:01:56,374 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:01:56,374 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:01:56,374 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => f41522ba5ab99eefcd7eeb6697805184, NAME => 'test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-28T07:01:56,374 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,374 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,375 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,375 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,376 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=10 2024-11-28T07:01:56,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 64c03d436be892d03f4313bba416e19f, server=1cb9cf437a36,44721,1732777312875 in 402 msec 2024-11-28T07:01:56,377 INFO [StoreOpener-f41522ba5ab99eefcd7eeb6697805184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,380 INFO [StoreOpener-f41522ba5ab99eefcd7eeb6697805184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f41522ba5ab99eefcd7eeb6697805184 columnFamilyName family 2024-11-28T07:01:56,380 DEBUG [StoreOpener-f41522ba5ab99eefcd7eeb6697805184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=64c03d436be892d03f4313bba416e19f, ASSIGN in 631 msec 2024-11-28T07:01:56,380 INFO [StoreOpener-f41522ba5ab99eefcd7eeb6697805184-1 {}] regionserver.HStore(327): Store=f41522ba5ab99eefcd7eeb6697805184/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,381 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,382 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4d610dd7e8db3ac925480924e91da676, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,382 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,383 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,384 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/6b0e706da5bc6b972222c767b7eb31f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,385 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,385 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,385 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 6b0e706da5bc6b972222c767b7eb31f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70181435, jitterRate=0.04578487575054169}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,385 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:01:56,385 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 6b0e706da5bc6b972222c767b7eb31f6: Running coprocessor pre-open hook at 1732777316356Writing region info on filesystem at 1732777316356Initializing all the Stores at 1732777316359 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316359Cleaning up temporary data from old regions at 1732777316373 (+14 ms)Running coprocessor post-open hooks at 1732777316385 (+12 ms)Region opened successfully at 1732777316385 2024-11-28T07:01:56,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:56,386 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6., pid=50, masterSystemTime=1732777316172 2024-11-28T07:01:56,388 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,389 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,392 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:01:56,392 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/f41522ba5ab99eefcd7eeb6697805184/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,392 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:01:56,392 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:01:56,392 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => c86fc11a5bf22fc0b32716e9c44823d3, NAME => 'test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-28T07:01:56,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2857a2dc2b09c2ea3bb5615768b39d67, server=1cb9cf437a36,38403,1732777312808 because future has completed 2024-11-28T07:01:56,393 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,393 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,393 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,393 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,393 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened f41522ba5ab99eefcd7eeb6697805184; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71681470, jitterRate=0.06813713908195496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,393 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:01:56,393 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for f41522ba5ab99eefcd7eeb6697805184: Running coprocessor pre-open hook at 1732777316375Writing region info on filesystem at 1732777316375Initializing all the Stores at 1732777316376 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316376Cleaning up temporary data from old regions at 1732777316385 (+9 ms)Running coprocessor post-open hooks at 1732777316393 (+8 ms)Region opened successfully at 1732777316393 2024-11-28T07:01:56,395 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184., pid=48, masterSystemTime=1732777316127 2024-11-28T07:01:56,397 INFO [StoreOpener-c86fc11a5bf22fc0b32716e9c44823d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,397 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:01:56,398 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:01:56,398 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:01:56,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=17 2024-11-28T07:01:56,398 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 8db2bf432be6aeda43c73169f2d030a4, NAME => 'test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-28T07:01:56,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 2857a2dc2b09c2ea3bb5615768b39d67, server=1cb9cf437a36,38403,1732777312808 in 321 msec 2024-11-28T07:01:56,399 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,399 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,399 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,399 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,399 INFO [StoreOpener-c86fc11a5bf22fc0b32716e9c44823d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c86fc11a5bf22fc0b32716e9c44823d3 columnFamilyName family 2024-11-28T07:01:56,399 DEBUG [StoreOpener-c86fc11a5bf22fc0b32716e9c44823d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,400 INFO [StoreOpener-c86fc11a5bf22fc0b32716e9c44823d3-1 {}] regionserver.HStore(327): Store=c86fc11a5bf22fc0b32716e9c44823d3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,401 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2857a2dc2b09c2ea3bb5615768b39d67, ASSIGN in 653 msec 2024-11-28T07:01:56,402 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,402 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,403 INFO [StoreOpener-8db2bf432be6aeda43c73169f2d030a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,403 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=f41522ba5ab99eefcd7eeb6697805184, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,403 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,403 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,406 INFO [StoreOpener-8db2bf432be6aeda43c73169f2d030a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8db2bf432be6aeda43c73169f2d030a4 columnFamilyName family 2024-11-28T07:01:56,406 DEBUG [StoreOpener-8db2bf432be6aeda43c73169f2d030a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,407 INFO [StoreOpener-8db2bf432be6aeda43c73169f2d030a4-1 {}] regionserver.HStore(327): Store=8db2bf432be6aeda43c73169f2d030a4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,407 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,408 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,408 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,408 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,409 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,409 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure f41522ba5ab99eefcd7eeb6697805184, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:56,411 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,411 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/c86fc11a5bf22fc0b32716e9c44823d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,412 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened c86fc11a5bf22fc0b32716e9c44823d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68934403, jitterRate=0.027202650904655457}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,412 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:01:56,413 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for c86fc11a5bf22fc0b32716e9c44823d3: Running coprocessor pre-open hook at 1732777316393Writing region info on filesystem at 1732777316393Initializing all the Stores at 1732777316395 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316395Cleaning up temporary data from old regions at 1732777316403 (+8 ms)Running coprocessor post-open hooks at 1732777316412 (+9 ms)Region opened successfully at 1732777316412 2024-11-28T07:01:56,414 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3., pid=54, masterSystemTime=1732777316172 2024-11-28T07:01:56,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b0e706da5bc6b972222c767b7eb31f6, server=1cb9cf437a36,38403,1732777312808 because future has completed 2024-11-28T07:01:56,417 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/8db2bf432be6aeda43c73169f2d030a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,417 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:01:56,418 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:01:56,418 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:01:56,418 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c79f2e3a60088e6bfee9be7f58096e9, NAME => 'test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-28T07:01:56,418 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 8db2bf432be6aeda43c73169f2d030a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60189795, jitterRate=-0.10310216248035431}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,418 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:01:56,418 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,418 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,418 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 8db2bf432be6aeda43c73169f2d030a4: Running coprocessor pre-open hook at 1732777316399Writing region info on filesystem at 1732777316399Initializing all the Stores at 1732777316400 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316400Cleaning up temporary data from old regions at 1732777316409 (+9 ms)Running coprocessor post-open hooks at 1732777316418 (+9 ms)Region opened successfully at 1732777316418 2024-11-28T07:01:56,418 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,418 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,420 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4., pid=41, masterSystemTime=1732777316127 2024-11-28T07:01:56,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=12 2024-11-28T07:01:56,421 INFO [StoreOpener-4c79f2e3a60088e6bfee9be7f58096e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure f41522ba5ab99eefcd7eeb6697805184, server=1cb9cf437a36,42757,1732777312688 in 382 msec 2024-11-28T07:01:56,423 INFO [StoreOpener-4c79f2e3a60088e6bfee9be7f58096e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c79f2e3a60088e6bfee9be7f58096e9 columnFamilyName family 2024-11-28T07:01:56,423 DEBUG [StoreOpener-4c79f2e3a60088e6bfee9be7f58096e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,424 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:01:56,424 INFO [StoreOpener-4c79f2e3a60088e6bfee9be7f58096e9-1 {}] regionserver.HStore(327): Store=4c79f2e3a60088e6bfee9be7f58096e9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,424 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:01:56,424 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN in 676 msec 2024-11-28T07:01:56,426 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,426 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,426 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=8db2bf432be6aeda43c73169f2d030a4, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,427 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,427 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,429 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:56,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,433 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4c79f2e3a60088e6bfee9be7f58096e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure c86fc11a5bf22fc0b32716e9c44823d3, server=1cb9cf437a36,38403,1732777312808 because future has completed 2024-11-28T07:01:56,434 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 4c79f2e3a60088e6bfee9be7f58096e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62095143, jitterRate=-0.07471026480197906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,434 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:01:56,434 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 4c79f2e3a60088e6bfee9be7f58096e9: Running coprocessor pre-open hook at 1732777316419Writing region info on filesystem at 1732777316419Initializing all the Stores at 1732777316420 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316420Cleaning up temporary data from old regions at 1732777316427 (+7 ms)Running coprocessor post-open hooks at 1732777316434 (+7 ms)Region opened successfully at 1732777316434 2024-11-28T07:01:56,435 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9., pid=45, masterSystemTime=1732777316172 2024-11-28T07:01:56,437 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:01:56,437 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:01:56,438 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:01:56,438 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 46d26e2a6997699edcc98e759a48e66a, NAME => 'test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-28T07:01:56,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=24 2024-11-28T07:01:56,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,42757,1732777312688 in 444 msec 2024-11-28T07:01:56,438 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,438 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:56,438 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,438 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,441 INFO [StoreOpener-46d26e2a6997699edcc98e759a48e66a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN in 693 msec 2024-11-28T07:01:56,442 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,442 INFO [StoreOpener-46d26e2a6997699edcc98e759a48e66a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46d26e2a6997699edcc98e759a48e66a columnFamilyName family 2024-11-28T07:01:56,442 DEBUG [StoreOpener-46d26e2a6997699edcc98e759a48e66a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:56,443 INFO [StoreOpener-46d26e2a6997699edcc98e759a48e66a-1 {}] regionserver.HStore(327): Store=46d26e2a6997699edcc98e759a48e66a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:56,443 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,445 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,446 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c79f2e3a60088e6bfee9be7f58096e9, server=1cb9cf437a36,38403,1732777312808 because future has completed 2024-11-28T07:01:56,446 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,447 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,448 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=14 2024-11-28T07:01:56,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure c86fc11a5bf22fc0b32716e9c44823d3, server=1cb9cf437a36,38403,1732777312808 in 376 msec 2024-11-28T07:01:56,451 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/46d26e2a6997699edcc98e759a48e66a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:01:56,452 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 46d26e2a6997699edcc98e759a48e66a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62042922, jitterRate=-0.07548841834068298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:56,452 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:01:56,452 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 46d26e2a6997699edcc98e759a48e66a: Running coprocessor pre-open hook at 1732777316439Writing region info on filesystem at 1732777316439Initializing all the Stores at 1732777316440 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777316440Cleaning up temporary data from old regions at 1732777316447 (+7 ms)Running coprocessor post-open hooks at 1732777316452 (+5 ms)Region opened successfully at 1732777316452 2024-11-28T07:01:56,453 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a., pid=56, masterSystemTime=1732777316172 2024-11-28T07:01:56,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c86fc11a5bf22fc0b32716e9c44823d3, ASSIGN in 706 msec 2024-11-28T07:01:56,456 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:01:56,456 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:01:56,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=29 2024-11-28T07:01:56,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c79f2e3a60088e6bfee9be7f58096e9, server=1cb9cf437a36,38403,1732777312808 in 437 msec 2024-11-28T07:01:56,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4c79f2e3a60088e6bfee9be7f58096e9, ASSIGN in 713 msec 2024-11-28T07:01:56,462 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=test, region=32789c56ce6b6f4105a05768e5d8e631. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T07:01:56,462 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=test, region=6b0e706da5bc6b972222c767b7eb31f6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T07:01:56,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:01:56,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46d26e2a6997699edcc98e759a48e66a, server=1cb9cf437a36,38403,1732777312808 because future has completed 2024-11-28T07:01:56,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=20 2024-11-28T07:01:56,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b0e706da5bc6b972222c767b7eb31f6, server=1cb9cf437a36,38403,1732777312808 in 413 msec 2024-11-28T07:01:56,474 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1cb9cf437a36,44721,1732777312875, table=test, region=24cb8e2edb958f7d862266464b772c73. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T07:01:56,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6b0e706da5bc6b972222c767b7eb31f6, ASSIGN in 726 msec 2024-11-28T07:01:56,478 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=1cb9cf437a36,42757,1732777312688, table=test, region=4d610dd7e8db3ac925480924e91da676. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T07:01:56,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=23 2024-11-28T07:01:56,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 46d26e2a6997699edcc98e759a48e66a, server=1cb9cf437a36,38403,1732777312808 in 397 msec 2024-11-28T07:01:56,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=46d26e2a6997699edcc98e759a48e66a, ASSIGN in 736 msec 2024-11-28T07:01:56,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=26 2024-11-28T07:01:56,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,42757,1732777312688 in 510 msec 2024-11-28T07:01:56,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN in 742 msec 2024-11-28T07:01:56,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=8 2024-11-28T07:01:56,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 32789c56ce6b6f4105a05768e5d8e631, server=1cb9cf437a36,38403,1732777312808 in 432 msec 2024-11-28T07:01:56,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=32789c56ce6b6f4105a05768e5d8e631, ASSIGN in 747 msec 2024-11-28T07:01:56,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=89adcb353365cff3d8866fb113c54483, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:56,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89adcb353365cff3d8866fb113c54483, server=1cb9cf437a36,44721,1732777312875 because future has completed 2024-11-28T07:01:56,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=25 2024-11-28T07:01:56,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 24cb8e2edb958f7d862266464b772c73, server=1cb9cf437a36,44721,1732777312875 in 543 msec 2024-11-28T07:01:56,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=24cb8e2edb958f7d862266464b772c73, ASSIGN in 756 msec 2024-11-28T07:01:56,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=19 2024-11-28T07:01:56,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 89adcb353365cff3d8866fb113c54483, server=1cb9cf437a36,44721,1732777312875 in 563 msec 2024-11-28T07:01:56,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=89adcb353365cff3d8866fb113c54483, ASSIGN in 763 msec 2024-11-28T07:01:56,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN in 765 msec 2024-11-28T07:01:56,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:56,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:56,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=7 2024-11-28T07:01:56,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,42757,1732777312688 in 493 msec 2024-11-28T07:01:56,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=4 2024-11-28T07:01:56,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN in 775 msec 2024-11-28T07:01:56,524 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T07:01:56,524 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732777316524"}]},"ts":"1732777316524"} 2024-11-28T07:01:56,527 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-28T07:01:56,529 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T07:01:56,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.2150 sec 2024-11-28T07:01:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:01:57,487 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-28T07:01:57,487 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-28T07:01:57,489 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:01:57,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-28T07:01:57,515 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:01:57,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-28T07:01:57,517 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:01:57,522 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-28T07:01:57,549 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-28T07:01:57,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:01:57,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:01:57,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:01:57,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:01:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 86c81a2b73bbd41aa0789229142fc551 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 32789c56ce6b6f4105a05768e5d8e631 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3d6a3aee8d008ffd9b93917e3b77bf62 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c86fc11a5bf22fc0b32716e9c44823d3 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2857a2dc2b09c2ea3bb5615768b39d67 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6b0e706da5bc6b972222c767b7eb31f6 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 46d26e2a6997699edcc98e759a48e66a changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5480f78d30eedd25c01f5f1822c0d5d1 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4c79f2e3a60088e6bfee9be7f58096e9 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f5a06150eff3d66689f377c8cac0f8ad changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 93afe67b3b08d7a0c25e98ea9a0dc636 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f41522ba5ab99eefcd7eeb6697805184 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fbb63e696a2f079596ba1efcff5d041b changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fa7f0c541fab82c2773800078745e787 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fa65e80f1531c2a3d21843d8e252160f changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8db2bf432be6aeda43c73169f2d030a4 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4d610dd7e8db3ac925480924e91da676 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0519f109d08233033823877d6faba4e1 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4276658fc73709d30ba28565a9fbbf03 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 64c03d436be892d03f4313bba416e19f changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 71bd07899358e3652d4466cdb1dd3176 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b32dc238678e85db1609526ddf98ddb1 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 89adcb353365cff3d8866fb113c54483 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1471a7fac83428eb937e98338e4948d6 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 24cb8e2edb958f7d862266464b772c73 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b6744df8a48cf582efed97c8fb9152b5 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:01:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-28T07:01:57,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-28T07:01:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-28T07:01:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T07:01:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-28T07:01:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-11-28T07:01:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(303): server 2 has 8 regions 2024-11-28T07:01:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:01:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:01:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-28T07:01:57,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:01:57,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:01:57,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-28T07:01:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-28T07:01:57,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-28T07:01:57,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-28T07:01:58,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 484 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-28T07:01:58,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-28T07:01:58,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.HMaster(2172): balance hri=fa65e80f1531c2a3d21843d8e252160f, source=1cb9cf437a36,42757,1732777312688, destination=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:01:58,122 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:01:58,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=fa65e80f1531c2a3d21843d8e252160f, regionState=CLOSING, regionLocation=1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:58,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE because future has completed 2024-11-28T07:01:58,128 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:01:58,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa65e80f1531c2a3d21843d8e252160f, server=1cb9cf437a36,42757,1732777312688}] 2024-11-28T07:01:58,285 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,285 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:01:58,286 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing fa65e80f1531c2a3d21843d8e252160f, disabling compactions & flushes 2024-11-28T07:01:58,286 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:58,286 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:58,286 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. after waiting 0 ms 2024-11-28T07:01:58,286 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:58,291 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa65e80f1531c2a3d21843d8e252160f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:01:58,294 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:58,295 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for fa65e80f1531c2a3d21843d8e252160f: Waiting for close lock at 1732777318286Running coprocessor pre-close hooks at 1732777318286Disabling compacts and flushes for region at 1732777318286Disabling writes for close at 1732777318286Writing region close event to WAL at 1732777318287 (+1 ms)Running coprocessor post-close hooks at 1732777318292 (+5 ms)Closed at 1732777318294 (+2 ms) 2024-11-28T07:01:58,295 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding fa65e80f1531c2a3d21843d8e252160f move to 1cb9cf437a36,44721,1732777312875 record at close sequenceid=2 2024-11-28T07:01:58,299 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=fa65e80f1531c2a3d21843d8e252160f, regionState=CLOSED 2024-11-28T07:01:58,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa65e80f1531c2a3d21843d8e252160f, server=1cb9cf437a36,42757,1732777312688 because future has completed 2024-11-28T07:01:58,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-28T07:01:58,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure fa65e80f1531c2a3d21843d8e252160f, server=1cb9cf437a36,42757,1732777312688 in 176 msec 2024-11-28T07:01:58,307 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE; state=CLOSED, location=1cb9cf437a36,44721,1732777312875; forceNewPlan=false, retain=false 2024-11-28T07:01:58,457 INFO [1cb9cf437a36:40243 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T07:01:58,458 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=fa65e80f1531c2a3d21843d8e252160f, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:58,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE because future has completed 2024-11-28T07:01:58,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa65e80f1531c2a3d21843d8e252160f, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:01:58,624 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:58,625 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => fa65e80f1531c2a3d21843d8e252160f, NAME => 'test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-28T07:01:58,625 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,626 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:58,626 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,626 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,630 INFO [StoreOpener-fa65e80f1531c2a3d21843d8e252160f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,632 INFO [StoreOpener-fa65e80f1531c2a3d21843d8e252160f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa65e80f1531c2a3d21843d8e252160f columnFamilyName family 2024-11-28T07:01:58,632 DEBUG [StoreOpener-fa65e80f1531c2a3d21843d8e252160f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:58,633 INFO [StoreOpener-fa65e80f1531c2a3d21843d8e252160f-1 {}] regionserver.HStore(327): Store=fa65e80f1531c2a3d21843d8e252160f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:01:58,633 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,635 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,637 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,637 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,637 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,640 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,641 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened fa65e80f1531c2a3d21843d8e252160f; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60844104, jitterRate=-0.09335219860076904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:01:58,641 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:01:58,642 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for fa65e80f1531c2a3d21843d8e252160f: Running coprocessor pre-open hook at 1732777318627Writing region info on filesystem at 1732777318627Initializing all the Stores at 1732777318629 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777318630 (+1 ms)Cleaning up temporary data from old regions at 1732777318637 (+7 ms)Running coprocessor post-open hooks at 1732777318641 (+4 ms)Region opened successfully at 1732777318641 2024-11-28T07:01:58,643 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f., pid=59, masterSystemTime=1732777318615 2024-11-28T07:01:58,645 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:58,645 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:01:58,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, openSeqNum=5, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:58,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa65e80f1531c2a3d21843d8e252160f, server=1cb9cf437a36,44721,1732777312875 because future has completed 2024-11-28T07:01:58,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-28T07:01:58,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure fa65e80f1531c2a3d21843d8e252160f, server=1cb9cf437a36,44721,1732777312875 in 189 msec 2024-11-28T07:01:58,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE in 532 msec 2024-11-28T07:01:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-28T07:01:58,725 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:01:58,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-28T07:01:58,748 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=381, OpenFileDescriptor=642, MaxFileDescriptor=1048576, SystemLoadAverage=122, ProcessCount=11, AvailableMemoryMB=8141 2024-11-28T07:01:58,751 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-28T07:01:58,752 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-28T07:01:58,752 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:58,752 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 1cb9cf437a36,42757,1732777312688: Simulated kill ***** 2024-11-28T07:01:58,753 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-28T07:01:58,755 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-28T07:01:58,757 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-28T07:01:58,757 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-28T07:01:58,757 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-28T07:01:58,758 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 336276032 }, "NonHeapMemoryUsage": { "committed": 153944064, "init": 7667712, "max": -1, "used": 151524392 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "1cb9cf437a36" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "1cb9cf437a36" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "1cb9cf437a36" } ] } 2024-11-28T07:01:58,760 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] master.MasterRpcServices(700): 1cb9cf437a36,42757,1732777312688 reported a fatal error: ***** ABORTING region server 1cb9cf437a36,42757,1732777312688: Simulated kill ***** 2024-11-28T07:01:58,762 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1cb9cf437a36,42757,1732777312688' ***** 2024-11-28T07:01:58,762 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-28T07:01:58,763 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T07:01:58,763 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T07:01:58,763 INFO [RS:0;1cb9cf437a36:42757 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-28T07:01:58,763 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:01:58,763 INFO [RS:0;1cb9cf437a36:42757 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-28T07:01:58,763 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:01:58,763 INFO [RS:0;1cb9cf437a36:42757 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1cb9cf437a36:42757. 2024-11-28T07:01:58,764 DEBUG [RS:0;1cb9cf437a36:42757 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:01:58,768 DEBUG [RS:0;1cb9cf437a36:42757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:01:58,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741836_1012 (size=33749) 2024-11-28T07:01:58,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741836_1012 (size=33749) 2024-11-28T07:01:58,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741836_1012 (size=33749) 2024-11-28T07:01:58,775 INFO [regionserver/1cb9cf437a36:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:01:58,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741835_1011 (size=2774) 2024-11-28T07:01:58,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741835_1011 (size=2774) 2024-11-28T07:01:58,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741835_1011 (size=2774) 2024-11-28T07:01:58,778 DEBUG [RS:0;1cb9cf437a36:42757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:01:58,778 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:01:58,778 INFO [RS:0;1cb9cf437a36:42757 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42757 2024-11-28T07:01:58,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1cb9cf437a36,42757,1732777312688 2024-11-28T07:01:58,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:01:58,829 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:01:58,839 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1cb9cf437a36,42757,1732777312688] 2024-11-28T07:01:58,850 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1cb9cf437a36,42757,1732777312688 already deleted, retry=false 2024-11-28T07:01:58,851 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1cb9cf437a36,42757,1732777312688 on 1cb9cf437a36,40243,1732777311941 2024-11-28T07:01:58,856 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:01:58,861 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:01:58,861 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 1cb9cf437a36,42757,1732777312688 (carryingMeta=true) 1cb9cf437a36,42757,1732777312688/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@f04771f[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-28T07:01:58,862 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true, isMeta: true 2024-11-28T07:01:58,864 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:01:58,865 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting 2024-11-28T07:01:58,866 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1cb9cf437a36,42757,1732777312688 WAL count=1, meta=true 2024-11-28T07:01:58,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta}] 2024-11-28T07:01:58,875 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:58,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:58,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:01:58,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42757-0x10181890e5a0001, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:01:58,939 INFO [RS:0;1cb9cf437a36:42757 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:01:58,939 INFO [RS:0;1cb9cf437a36:42757 {}] regionserver.HRegionServer(1031): Exiting; stopping=1cb9cf437a36,42757,1732777312688; zookeeper connection closed. 2024-11-28T07:01:58,939 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@18a4bec5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@18a4bec5 2024-11-28T07:01:59,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38403 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-28T07:01:59,060 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, size=33.0 K (33749bytes) 2024-11-28T07:01:59,061 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-28T07:01:59,061 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-28T07:01:59,061 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta 2024-11-28T07:01:59,066 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta after 4ms 2024-11-28T07:01:59,073 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta took 14ms 2024-11-28T07:01:59,083 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-28T07:01:59,089 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta.temp 2024-11-28T07:01:59,093 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta.temp 2024-11-28T07:01:59,096 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta so closing down 2024-11-28T07:01:59,096 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-28T07:01:59,097 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-28T07:01:59,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741864_1040 (size=33508) 2024-11-28T07:01:59,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741864_1040 (size=33508) 2024-11-28T07:01:59,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741864_1040 (size=33508) 2024-11-28T07:01:59,113 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-28T07:01:59,116 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta.temp to hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-28T07:01:59,121 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-28T07:01:59,121 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, journal: Splitting hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, size=33.0 K (33749bytes) at 1732777319060Creating recovered edits writer path=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta.temp at 1732777319093 (+33 ms)Finishing writing output for hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta so closing down at 1732777319096 (+3 ms)3 split writer threads finished at 1732777319097 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1732777319113 (+16 ms)Rename recovered edits hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta.temp to hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732777319116 (+3 ms)Processed 62 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732777319121 (+5 ms) 2024-11-28T07:01:59,121 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta 2024-11-28T07:01:59,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-28T07:01:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-28T07:01:59,132 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta to hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/oldWALs 2024-11-28T07:01:59,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-28T07:01:59,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 in 255 msec 2024-11-28T07:01:59,137 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:59,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-28T07:01:59,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 in 269 msec 2024-11-28T07:01:59,142 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting dir is empty, no logs to split. 2024-11-28T07:01:59,142 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1cb9cf437a36,42757,1732777312688 WAL count=0, meta=true 2024-11-28T07:01:59,142 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 1cb9cf437a36,42757,1732777312688 WAL splitting is done? wals=0, meta=true 2024-11-28T07:01:59,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T07:01:59,146 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:01:59,147 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:01:59,271 INFO [regionserver/1cb9cf437a36:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T07:01:59,271 INFO [regionserver/1cb9cf437a36:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T07:01:59,298 DEBUG [1cb9cf437a36:40243 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-28T07:01:59,298 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T07:01:59,299 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T07:01:59,299 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T07:01:59,299 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:01:59,299 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:01:59,299 INFO [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:01:59,299 INFO [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:01:59,299 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T07:01:59,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:59,301 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPENING 2024-11-28T07:01:59,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:59,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:59,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:59,338 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:01:59,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:59,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:59,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:59,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:01:59,497 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T07:01:59,497 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:01:59,497 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T07:01:59,500 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C38403%2C1732777312808.meta, suffix=.meta, logDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,38403,1732777312808, archiveDir=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/oldWALs, maxLogs=32 2024-11-28T07:01:59,514 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,38403,1732777312808/1cb9cf437a36%2C38403%2C1732777312808.meta.1732777319501.meta, exclude list is [], retry=0 2024-11-28T07:01:59,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38317,DS-c4fa8ba8-6773-47fb-a54b-615163e7b781,DISK] 2024-11-28T07:01:59,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38193,DS-cd3e3d2e-47a2-4f27-a3cb-ae1857b37466,DISK] 2024-11-28T07:01:59,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33227,DS-26f14f1e-6f3e-493a-a227-da6fa5ec324f,DISK] 2024-11-28T07:01:59,519 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,38403,1732777312808/1cb9cf437a36%2C38403%2C1732777312808.meta.1732777319501.meta 2024-11-28T07:01:59,520 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36335:36335),(127.0.0.1/127.0.0.1:43245:43245),(127.0.0.1/127.0.0.1:40643:40643)] 2024-11-28T07:01:59,520 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:01:59,520 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T07:01:59,521 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T07:01:59,521 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T07:01:59,521 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T07:01:59,521 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:01:59,521 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T07:01:59,521 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T07:01:59,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T07:01:59,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:01:59,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:59,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:59,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T07:01:59,528 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:01:59,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:59,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:59,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T07:01:59,531 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:01:59,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:59,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:59,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T07:01:59,533 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:01:59,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:01:59,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:01:59,533 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:01:59,534 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740 2024-11-28T07:01:59,537 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740 2024-11-28T07:01:59,538 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-28T07:01:59,559 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-28T07:01:59,563 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-28T07:01:59,630 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/info/b4d25b201cd74f04860da0fb6e0640b8 is 138, key is test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03./info:regioninfo/1732777316252/Put/seqid=0 2024-11-28T07:01:59,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741866_1042 (size=36371) 2024-11-28T07:01:59,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741866_1042 (size=36371) 2024-11-28T07:01:59,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741866_1042 (size=36371) 2024-11-28T07:01:59,643 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/info/b4d25b201cd74f04860da0fb6e0640b8 2024-11-28T07:01:59,706 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/ns/af638009f3d8464e8e0bf22ea0067dc0 is 43, key is default/ns:d/1732777315065/Put/seqid=0 2024-11-28T07:01:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741867_1043 (size=5153) 2024-11-28T07:01:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741867_1043 (size=5153) 2024-11-28T07:01:59,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741867_1043 (size=5153) 2024-11-28T07:01:59,715 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/ns/af638009f3d8464e8e0bf22ea0067dc0 2024-11-28T07:01:59,738 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/table/dd112d201d50441488cde9cdd9513b46 is 40, key is test/table:state/1732777316524/Put/seqid=0 2024-11-28T07:01:59,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741868_1044 (size=5165) 2024-11-28T07:01:59,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741868_1044 (size=5165) 2024-11-28T07:01:59,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741868_1044 (size=5165) 2024-11-28T07:01:59,747 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/table/dd112d201d50441488cde9cdd9513b46 2024-11-28T07:01:59,756 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/info/b4d25b201cd74f04860da0fb6e0640b8 as hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/info/b4d25b201cd74f04860da0fb6e0640b8 2024-11-28T07:01:59,766 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/info/b4d25b201cd74f04860da0fb6e0640b8, entries=273, sequenceid=64, filesize=35.5 K 2024-11-28T07:01:59,769 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/ns/af638009f3d8464e8e0bf22ea0067dc0 as hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/ns/af638009f3d8464e8e0bf22ea0067dc0 2024-11-28T07:01:59,778 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/ns/af638009f3d8464e8e0bf22ea0067dc0, entries=2, sequenceid=64, filesize=5.0 K 2024-11-28T07:01:59,780 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/table/dd112d201d50441488cde9cdd9513b46 as hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/table/dd112d201d50441488cde9cdd9513b46 2024-11-28T07:01:59,790 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/table/dd112d201d50441488cde9cdd9513b46, entries=2, sequenceid=64, filesize=5.0 K 2024-11-28T07:01:59,795 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 231ms, sequenceid=64, compaction requested=false; wal=null 2024-11-28T07:01:59,797 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T07:01:59,800 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-28T07:01:59,802 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:01:59,802 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:01:59,803 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:01:59,804 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:01:59,807 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-28T07:01:59,808 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64843673, jitterRate=-0.033753976225852966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:01:59,808 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T07:01:59,808 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732777319521Writing region info on filesystem at 1732777319521Initializing all the Stores at 1732777319523 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777319523Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777319524 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777319524Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777319524Obtaining lock to block concurrent updates at 1732777319563 (+39 ms)Preparing flush snapshotting stores in 1588230740 at 1732777319563Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732777319567 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732777319567Flushing 1588230740/info: creating writer at 1732777319569 (+2 ms)Flushing 1588230740/info: appending metadata at 1732777319620 (+51 ms)Flushing 1588230740/info: closing flushed file at 1732777319622 (+2 ms)Flushing 1588230740/ns: creating writer at 1732777319689 (+67 ms)Flushing 1588230740/ns: appending metadata at 1732777319705 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732777319705Flushing 1588230740/table: creating writer at 1732777319722 (+17 ms)Flushing 1588230740/table: appending metadata at 1732777319738 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732777319738Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6862e439: reopening flushed file at 1732777319755 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ada0ae1: reopening flushed file at 1732777319767 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a1d3785: reopening flushed file at 1732777319779 (+12 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 231ms, sequenceid=64, compaction requested=false; wal=null at 1732777319795 (+16 ms)Cleaning up temporary data from old regions at 1732777319802 (+7 ms)Running coprocessor post-open hooks at 1732777319808 (+6 ms)Region opened successfully at 1732777319808 2024-11-28T07:01:59,810 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732777319492 2024-11-28T07:01:59,813 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T07:01:59,813 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T07:01:59,814 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:59,814 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:01:59,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:59,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:59,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:01:59,862 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:01:59,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:59,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:59,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:01:59,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-28T07:01:59,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 in 524 msec 2024-11-28T07:01:59,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-28T07:01:59,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 722 msec 2024-11-28T07:01:59,871 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 1cb9cf437a36,42757,1732777312688 had 9 regions 2024-11-28T07:01:59,873 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true, isMeta: false 2024-11-28T07:01:59,875 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1cb9cf437a36,42757,1732777312688 WAL count=1, meta=false 2024-11-28T07:01:59,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425}] 2024-11-28T07:01:59,877 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:01:59,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:02:00,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-28T07:02:00,047 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425, size=2.7 K (2774bytes) 2024-11-28T07:02:00,047 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425 2024-11-28T07:02:00,048 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425 after 1ms 2024-11-28T07:02:00,051 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425 took 4ms 2024-11-28T07:02:00,055 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for fa7f0c541fab82c2773800078745e787: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-28T07:02:00,057 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 0519f109d08233033823877d6faba4e1: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-28T07:02:00,059 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f5a06150eff3d66689f377c8cac0f8ad: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-28T07:02:00,061 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for fa65e80f1531c2a3d21843d8e252160f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-28T07:02:00,063 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 93afe67b3b08d7a0c25e98ea9a0dc636: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-28T07:02:00,065 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for fbb63e696a2f079596ba1efcff5d041b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-28T07:02:00,066 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 4d610dd7e8db3ac925480924e91da676: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-28T07:02:00,068 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f41522ba5ab99eefcd7eeb6697805184: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-28T07:02:00,069 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8db2bf432be6aeda43c73169f2d030a4: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-28T07:02:00,070 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425 so closing down 2024-11-28T07:02:00,070 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-28T07:02:00,070 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-28T07:02:00,070 INFO [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-28T07:02:00,070 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425, journal: Splitting hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425, size=2.7 K (2774bytes) at 1732777320047Finishing writing output for hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425 so closing down at 1732777320070 (+23 ms)3 split writer threads finished at 1732777320070Processed 10 edits across 0 Regions in 18 ms; skipped=10; WAL=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1732777320070 2024-11-28T07:02:00,070 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425 2024-11-28T07:02:00,070 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-28T07:02:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-28T07:02:00,075 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting/1cb9cf437a36%2C42757%2C1732777312688.1732777314425 to hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/oldWALs 2024-11-28T07:02:00,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-28T07:02:00,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 in 197 msec 2024-11-28T07:02:00,079 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-28T07:02:00,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 in 205 msec 2024-11-28T07:02:00,084 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/WALs/1cb9cf437a36,42757,1732777312688-splitting dir is empty, no logs to split. 2024-11-28T07:02:00,084 INFO [PEWorker-1 {}] master.SplitWALManager(105): 1cb9cf437a36,42757,1732777312688 WAL count=0, meta=false 2024-11-28T07:02:00,084 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 1cb9cf437a36,42757,1732777312688 WAL splitting is done? wals=0, meta=false 2024-11-28T07:02:00,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN}] 2024-11-28T07:02:00,088 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,089 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:02:00,090 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,091 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:02:00,091 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,092 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:02:00,093 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,094 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:02:00,094 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,095 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:02:00,096 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,096 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:02:00,097 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,098 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:02:00,099 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,099 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:02:00,240 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T07:02:00,241 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T07:02:00,241 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T07:02:00,241 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:02:00,241 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:02:00,241 INFO [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:02:00,242 INFO [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:02:00,242 DEBUG [1cb9cf437a36:40243 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T07:02:00,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=0519f109d08233033823877d6faba4e1, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,245 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1cb9cf437a36:42757 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:00,246 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, regionLocation=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,248 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=fa7f0c541fab82c2773800078745e787, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1, error=java.net.ConnectException: Call to address=1cb9cf437a36:42757 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,248 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, regionLocation=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-28T07:02:00,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 from cache 2024-11-28T07:02:00,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 1cb9cf437a36:42757 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,251 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1cb9cf437a36:42757 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:00,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1cb9cf437a36:42757 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1cb9cf437a36:42757 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1cb9cf437a36:42757 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1cb9cf437a36:42757 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1cb9cf437a36:42757 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1cb9cf437a36:42757 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,42757,1732777312688, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=1cb9cf437a36:42757 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 1cb9cf437a36:42757 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:42757 2024-11-28T07:02:00,355 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:02:00,355 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,38403,1732777312808, seqNum=-1] 2024-11-28T07:02:00,356 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:02:00,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54619, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:02:00,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN because future has completed 2024-11-28T07:02:00,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:02:00,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN because future has completed 2024-11-28T07:02:00,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN because future has completed 2024-11-28T07:02:00,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN because future has completed 2024-11-28T07:02:00,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN because future has completed 2024-11-28T07:02:00,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:02:00,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN because future has completed 2024-11-28T07:02:00,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN because future has completed 2024-11-28T07:02:00,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN because future has completed 2024-11-28T07:02:00,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:02:00,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:02:00,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:02:00,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,44721,1732777312875}] 2024-11-28T07:02:00,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:02:00,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f41522ba5ab99eefcd7eeb6697805184, server=1cb9cf437a36,38403,1732777312808}] 2024-11-28T07:02:00,404 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:02:00,462 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-28T07:02:00,518 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:02:00,518 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 93afe67b3b08d7a0c25e98ea9a0dc636, NAME => 'test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-28T07:02:00,519 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,519 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:02:00,519 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,519 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,521 INFO [StoreOpener-93afe67b3b08d7a0c25e98ea9a0dc636-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,523 INFO [StoreOpener-93afe67b3b08d7a0c25e98ea9a0dc636-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93afe67b3b08d7a0c25e98ea9a0dc636 columnFamilyName family 2024-11-28T07:02:00,523 DEBUG [StoreOpener-93afe67b3b08d7a0c25e98ea9a0dc636-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:02:00,524 INFO [StoreOpener-93afe67b3b08d7a0c25e98ea9a0dc636-1 {}] regionserver.HStore(327): Store=93afe67b3b08d7a0c25e98ea9a0dc636/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:02:00,524 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,525 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,525 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:02:00,525 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d610dd7e8db3ac925480924e91da676, NAME => 'test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-28T07:02:00,526 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,526 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:02:00,526 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,526 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,526 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,527 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,527 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,528 INFO [StoreOpener-4d610dd7e8db3ac925480924e91da676-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,529 INFO [StoreOpener-4d610dd7e8db3ac925480924e91da676-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d610dd7e8db3ac925480924e91da676 columnFamilyName family 2024-11-28T07:02:00,529 DEBUG [StoreOpener-4d610dd7e8db3ac925480924e91da676-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:02:00,529 INFO [StoreOpener-4d610dd7e8db3ac925480924e91da676-1 {}] regionserver.HStore(327): Store=4d610dd7e8db3ac925480924e91da676/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:02:00,529 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,529 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,530 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 93afe67b3b08d7a0c25e98ea9a0dc636; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73833748, jitterRate=0.100208580493927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:02:00,530 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:00,530 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,530 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 93afe67b3b08d7a0c25e98ea9a0dc636: Running coprocessor pre-open hook at 1732777320519Writing region info on filesystem at 1732777320519Initializing all the Stores at 1732777320521 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777320521Cleaning up temporary data from old regions at 1732777320527 (+6 ms)Running coprocessor post-open hooks at 1732777320530 (+3 ms)Region opened successfully at 1732777320530 2024-11-28T07:02:00,531 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636., pid=79, masterSystemTime=1732777320513 2024-11-28T07:02:00,532 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,532 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,532 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,534 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:02:00,535 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:02:00,535 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:02:00,535 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 8db2bf432be6aeda43c73169f2d030a4, NAME => 'test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-28T07:02:00,535 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,535 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,536 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:02:00,536 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,536 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,536 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 4d610dd7e8db3ac925480924e91da676; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68377432, jitterRate=0.018903136253356934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:02:00,537 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:00,537 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 4d610dd7e8db3ac925480924e91da676: Running coprocessor pre-open hook at 1732777320526Writing region info on filesystem at 1732777320526Initializing all the Stores at 1732777320527 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777320527Cleaning up temporary data from old regions at 1732777320532 (+5 ms)Running coprocessor post-open hooks at 1732777320537 (+5 ms)Region opened successfully at 1732777320537 2024-11-28T07:02:00,537 INFO [StoreOpener-8db2bf432be6aeda43c73169f2d030a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,538 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676., pid=78, masterSystemTime=1732777320520 2024-11-28T07:02:00,538 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-28T07:02:00,538 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,538 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,540 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:02:00,540 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:02:00,541 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:02:00,541 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => f41522ba5ab99eefcd7eeb6697805184, NAME => 'test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-28T07:02:00,541 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,541 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:02:00,541 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,541 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,542 INFO [StoreOpener-8db2bf432be6aeda43c73169f2d030a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8db2bf432be6aeda43c73169f2d030a4 columnFamilyName family 2024-11-28T07:02:00,542 DEBUG [StoreOpener-8db2bf432be6aeda43c73169f2d030a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:02:00,542 INFO [StoreOpener-8db2bf432be6aeda43c73169f2d030a4-1 {}] regionserver.HStore(327): Store=8db2bf432be6aeda43c73169f2d030a4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:02:00,543 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,543 INFO [StoreOpener-f41522ba5ab99eefcd7eeb6697805184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,544 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,544 INFO [StoreOpener-f41522ba5ab99eefcd7eeb6697805184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f41522ba5ab99eefcd7eeb6697805184 columnFamilyName family 2024-11-28T07:02:00,544 DEBUG [StoreOpener-f41522ba5ab99eefcd7eeb6697805184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:02:00,545 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,545 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,545 INFO [StoreOpener-f41522ba5ab99eefcd7eeb6697805184-1 {}] regionserver.HStore(327): Store=f41522ba5ab99eefcd7eeb6697805184/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:02:00,545 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,545 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,546 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,547 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,548 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,549 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 8db2bf432be6aeda43c73169f2d030a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73719926, jitterRate=0.09851250052452087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:02:00,549 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:00,549 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 8db2bf432be6aeda43c73169f2d030a4: Running coprocessor pre-open hook at 1732777320536Writing region info on filesystem at 1732777320536Initializing all the Stores at 1732777320537 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777320537Cleaning up temporary data from old regions at 1732777320545 (+8 ms)Running coprocessor post-open hooks at 1732777320549 (+4 ms)Region opened successfully at 1732777320549 2024-11-28T07:02:00,549 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,549 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,550 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-28T07:02:00,552 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,552 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,552 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f41522ba5ab99eefcd7eeb6697805184, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,553 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4., pid=80, masterSystemTime=1732777320513 2024-11-28T07:02:00,553 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened f41522ba5ab99eefcd7eeb6697805184; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70996389, jitterRate=0.05792863667011261}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:02:00,553 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:00,553 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for f41522ba5ab99eefcd7eeb6697805184: Running coprocessor pre-open hook at 1732777320542Writing region info on filesystem at 1732777320542Initializing all the Stores at 1732777320543 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777320543Cleaning up temporary data from old regions at 1732777320549 (+6 ms)Running coprocessor post-open hooks at 1732777320553 (+4 ms)Region opened successfully at 1732777320553 2024-11-28T07:02:00,553 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f41522ba5ab99eefcd7eeb6697805184, server=1cb9cf437a36,38403,1732777312808 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,554 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184., pid=82, masterSystemTime=1732777320520 2024-11-28T07:02:00,556 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:02:00,556 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:02:00,556 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. 2024-11-28T07:02:00,556 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 0519f109d08233033823877d6faba4e1, NAME => 'test,zzz,1732777315307.0519f109d08233033823877d6faba4e1.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-28T07:02:00,557 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,557 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:02:00,557 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,zzz,1732777315307.0519f109d08233033823877d6faba4e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:02:00,557 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:02:00,557 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,557 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. 2024-11-28T07:02:00,557 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,557 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => fa7f0c541fab82c2773800078745e787, NAME => 'test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-28T07:02:00,557 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,557 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:02:00,558 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,558 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,559 INFO [StoreOpener-0519f109d08233033823877d6faba4e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,559 INFO [StoreOpener-fa7f0c541fab82c2773800078745e787-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,560 INFO [StoreOpener-0519f109d08233033823877d6faba4e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0519f109d08233033823877d6faba4e1 columnFamilyName family 2024-11-28T07:02:00,560 DEBUG [StoreOpener-0519f109d08233033823877d6faba4e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:02:00,560 INFO [StoreOpener-fa7f0c541fab82c2773800078745e787-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa7f0c541fab82c2773800078745e787 columnFamilyName family 2024-11-28T07:02:00,560 DEBUG [StoreOpener-fa7f0c541fab82c2773800078745e787-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:02:00,560 INFO [StoreOpener-0519f109d08233033823877d6faba4e1-1 {}] regionserver.HStore(327): Store=0519f109d08233033823877d6faba4e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:02:00,561 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,561 INFO [StoreOpener-fa7f0c541fab82c2773800078745e787-1 {}] regionserver.HStore(327): Store=fa7f0c541fab82c2773800078745e787/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:02:00,561 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,562 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,562 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,563 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,563 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,563 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,563 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,563 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,563 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,565 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,565 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,566 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 0519f109d08233033823877d6faba4e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62823806, jitterRate=-0.06385233998298645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:02:00,566 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0519f109d08233033823877d6faba4e1 2024-11-28T07:02:00,567 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened fa7f0c541fab82c2773800078745e787; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64119370, jitterRate=-0.044546931982040405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:02:00,567 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:00,567 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 0519f109d08233033823877d6faba4e1: Running coprocessor pre-open hook at 1732777320557Writing region info on filesystem at 1732777320557Initializing all the Stores at 1732777320558 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777320558Cleaning up temporary data from old regions at 1732777320563 (+5 ms)Running coprocessor post-open hooks at 1732777320567 (+4 ms)Region opened successfully at 1732777320567 2024-11-28T07:02:00,567 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for fa7f0c541fab82c2773800078745e787: Running coprocessor pre-open hook at 1732777320558Writing region info on filesystem at 1732777320558Initializing all the Stores at 1732777320559 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777320559Cleaning up temporary data from old regions at 1732777320563 (+4 ms)Running coprocessor post-open hooks at 1732777320567 (+4 ms)Region opened successfully at 1732777320567 2024-11-28T07:02:00,568 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787., pid=77, masterSystemTime=1732777320520 2024-11-28T07:02:00,568 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732777315307.0519f109d08233033823877d6faba4e1., pid=75, masterSystemTime=1732777320513 2024-11-28T07:02:00,568 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,568 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,568 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,569 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,569 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,569 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,569 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,570 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,571 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:00,571 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,577 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,578 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,579 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,580 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f41522ba5ab99eefcd7eeb6697805184, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f41522ba5ab99eefcd7eeb6697805184, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,587 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-28T07:02:00,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,589 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,589 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,593 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "1cb9cf437a36" port: 44721 start_code: 1732777312875 } transition { transition_code: OPENED region_info { region_id: 1732777315307 table_name { namespace: "default" qualifier: "test" } start_key: "zzz" end_key: "" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 75 initiating_master_active_time: 1732777312990 }; retry (#0) after 1007ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-28T07:02:00,593 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "1cb9cf437a36" port: 38403 start_code: 1732777312808 } transition { transition_code: OPENED region_info { region_id: 1732777315307 table_name { namespace: "default" qualifier: "test" } start_key: "nnn" end_key: "ooo" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1732777312990 }; retry (#0) after 1004ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-28T07:02:00,595 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,613 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,639 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,640 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,640 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,640 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,640 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN to state=OFFLINE, location=null, table=test, region=f41522ba5ab99eefcd7eeb6697805184 to restore RIT 2024-11-28T07:02:00,641 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350641 2024-11-28T07:02:00,641 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,641 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,643 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,644 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,644 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,644 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,38403,1732777312808 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,682 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:00,682 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777380682 2024-11-28T07:02:00,682 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777440682 2024-11-28T07:02:00,682 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 41 msec 2024-11-28T07:02:00,682 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,682 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,682 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,683 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,683 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,683 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,683 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,683 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,684 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:00,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,689 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,690 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,691 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,695 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,695 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,695 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,695 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,695 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,695 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,695 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,695 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a06150eff3d66689f377c8cac0f8ad, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,696 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-28T07:02:00,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,697 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,697 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,721 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN to state=OFFLINE, location=null, table=test, region=f41522ba5ab99eefcd7eeb6697805184 to restore RIT 2024-11-28T07:02:00,722 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350722 2024-11-28T07:02:00,722 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,722 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,723 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,725 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,44721,1732777312875 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,737 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:00,737 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777380737 2024-11-28T07:02:00,737 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777440737 2024-11-28T07:02:00,737 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-28T07:02:00,737 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,737 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,737 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,738 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,738 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,738 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,738 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,738 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,739 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T07:02:00,740 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,740 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,750 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,750 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,750 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db2bf432be6aeda43c73169f2d030a4, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,750 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-28T07:02:00,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,751 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,751 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,763 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN to state=OFFLINE, location=null, table=test, region=f41522ba5ab99eefcd7eeb6697805184 to restore RIT 2024-11-28T07:02:00,763 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350763 2024-11-28T07:02:00,763 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,763 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,765 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,767 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,767 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,767 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,775 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:00,775 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777380775 2024-11-28T07:02:00,775 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777440775 2024-11-28T07:02:00,776 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-28T07:02:00,776 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,776 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,776 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,776 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,776 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,776 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,776 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,776 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,777 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:00,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,781 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,787 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,787 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,787 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,787 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,787 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,787 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,787 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-28T07:02:00,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,788 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,788 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,796 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,796 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,796 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN to state=OFFLINE, location=null, table=test, region=f41522ba5ab99eefcd7eeb6697805184 to restore RIT 2024-11-28T07:02:00,797 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350797 2024-11-28T07:02:00,797 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,797 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,799 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,800 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,801 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,809 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:00,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777380809 2024-11-28T07:02:00,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777440809 2024-11-28T07:02:00,809 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-28T07:02:00,809 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,809 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,809 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,809 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,809 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,809 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,809 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,810 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:00,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,815 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,815 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,818 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,818 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,819 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,819 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,819 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,819 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,819 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,819 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,819 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d610dd7e8db3ac925480924e91da676, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,819 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-28T07:02:00,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,820 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,820 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN to state=OFFLINE, location=null, table=test, region=f41522ba5ab99eefcd7eeb6697805184 to restore RIT 2024-11-28T07:02:00,830 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350830 2024-11-28T07:02:00,830 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,830 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,832 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,833 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,833 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,833 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,841 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:00,842 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777380842 2024-11-28T07:02:00,842 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777440842 2024-11-28T07:02:00,842 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-28T07:02:00,842 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,842 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,842 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,842 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,842 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,842 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,842 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,843 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,844 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:00,843 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,844 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,846 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,849 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,849 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,850 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,850 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,850 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,851 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,851 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,851 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa7f0c541fab82c2773800078745e787, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,851 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-28T07:02:00,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,852 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,852 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN to state=OFFLINE, location=null, table=test, region=f41522ba5ab99eefcd7eeb6697805184 to restore RIT 2024-11-28T07:02:00,864 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350864 2024-11-28T07:02:00,864 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,864 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,866 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,869 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,869 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,869 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,869 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,873 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:00,873 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777380873 2024-11-28T07:02:00,873 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777440873 2024-11-28T07:02:00,873 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-28T07:02:00,873 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,873 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,873 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,873 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,873 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,874 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,874 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,874 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,875 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,877 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,878 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,880 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,881 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,881 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbb63e696a2f079596ba1efcff5d041b, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,881 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-28T07:02:00,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,882 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,883 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,892 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,892 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,892 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,893 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN to state=OFFLINE, location=null, table=test, region=f41522ba5ab99eefcd7eeb6697805184 to restore RIT 2024-11-28T07:02:00,894 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350894 2024-11-28T07:02:00,894 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,894 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,896 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,897 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,897 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,898 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,901 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,903 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:00,904 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777380903 2024-11-28T07:02:00,904 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777440904 2024-11-28T07:02:00,904 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-28T07:02:00,904 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,904 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,904 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,904 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,904 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,904 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,905 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,912 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,912 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,912 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0519f109d08233033823877d6faba4e1, server=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,912 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-28T07:02:00,913 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,913 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,913 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN to state=OFFLINE, location=null, table=test, region=f41522ba5ab99eefcd7eeb6697805184 to restore RIT 2024-11-28T07:02:00,922 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350922 2024-11-28T07:02:00,922 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,922 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,924 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,927 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,927 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,927 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,933 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:00,933 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777380933 2024-11-28T07:02:00,933 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777440933 2024-11-28T07:02:00,933 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-28T07:02:00,933 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,934 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,934 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,934 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,934 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,934 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,935 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,939 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,940 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,940 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,940 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,940 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,940 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,941 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,941 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,941 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,941 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,941 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,941 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,941 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,941 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN 2024-11-28T07:02:00,941 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-28T07:02:00,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,942 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,942 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,947 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,947 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,954 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,955 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,955 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,955 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,955 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f41522ba5ab99eefcd7eeb6697805184, ASSIGN to state=OFFLINE, location=null, table=test, region=f41522ba5ab99eefcd7eeb6697805184 to restore RIT 2024-11-28T07:02:00,955 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350955 2024-11-28T07:02:00,955 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,955 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,956 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,957 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,957 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,957 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,961 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,962 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,965 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:00,965 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777380965 2024-11-28T07:02:00,965 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777440965 2024-11-28T07:02:00,965 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-28T07:02:00,965 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:00,965 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,965 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:00,965 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:00,966 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:00,966 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:00,966 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:00,966 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:00,967 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:00,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:00,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,969 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:00,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:00,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:00,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:00,970 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:00,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:00,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,972 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,973 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,973 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,973 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,973 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,973 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:00,973 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,973 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,974 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:00,974 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN 2024-11-28T07:02:00,974 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-28T07:02:00,975 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:00,975 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:00,975 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:00,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:00,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,987 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:00,988 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:00,989 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:00,989 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0519f109d08233033823877d6faba4e1, ASSIGN to state=OFFLINE, location=null, table=test, region=0519f109d08233033823877d6faba4e1 to restore RIT 2024-11-28T07:02:00,989 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777350989 2024-11-28T07:02:00,989 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:00,989 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,991 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:00,992 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:00,992 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:00,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:00,992 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:00,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:00,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:00,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:00,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,000 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,000 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381000 2024-11-28T07:02:01,000 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441000 2024-11-28T07:02:01,000 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-28T07:02:01,000 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,000 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,000 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,000 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,001 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,001 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,001 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,001 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,002 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,004 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,004 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,004 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,004 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,004 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,004 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,004 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:01,005 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:01,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN 2024-11-28T07:02:01,010 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-28T07:02:01,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,011 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,011 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,020 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,020 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,020 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,021 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:01,021 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:01,021 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:01,021 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:01,021 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:01,021 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4d610dd7e8db3ac925480924e91da676, ASSIGN to state=OFFLINE, location=null, table=test, region=4d610dd7e8db3ac925480924e91da676 to restore RIT 2024-11-28T07:02:01,022 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351022 2024-11-28T07:02:01,022 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,022 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,023 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,027 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:01,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,032 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,032 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381032 2024-11-28T07:02:01,032 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441032 2024-11-28T07:02:01,032 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-28T07:02:01,032 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,032 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,032 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,032 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,033 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,033 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,033 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,033 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,034 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,036 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:01,037 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,037 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,040 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,040 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,040 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:01,040 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,040 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:01,041 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,041 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN 2024-11-28T07:02:01,041 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-28T07:02:01,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,042 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,042 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,050 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,051 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:01,051 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:01,051 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:01,051 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:01,051 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5a06150eff3d66689f377c8cac0f8ad, ASSIGN to state=OFFLINE, location=null, table=test, region=f5a06150eff3d66689f377c8cac0f8ad to restore RIT 2024-11-28T07:02:01,051 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351051 2024-11-28T07:02:01,051 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,052 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,053 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,056 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,056 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:01,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,056 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,059 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,059 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,059 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381059 2024-11-28T07:02:01,059 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441059 2024-11-28T07:02:01,059 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-28T07:02:01,059 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,059 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,059 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,060 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,060 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,060 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,060 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,060 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,061 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,062 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:01,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,064 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:01,065 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,065 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN 2024-11-28T07:02:01,065 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-28T07:02:01,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,066 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,066 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,076 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:01,076 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:01,076 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:01,076 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fa7f0c541fab82c2773800078745e787, ASSIGN to state=OFFLINE, location=null, table=test, region=fa7f0c541fab82c2773800078745e787 to restore RIT 2024-11-28T07:02:01,077 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351077 2024-11-28T07:02:01,077 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,077 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,078 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,079 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,080 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:01,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,080 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,085 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,085 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381085 2024-11-28T07:02:01,085 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441085 2024-11-28T07:02:01,085 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-28T07:02:01,085 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,085 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,086 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,086 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,086 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,086 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,086 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,086 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,087 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:01,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,092 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,093 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,093 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,093 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,093 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN 2024-11-28T07:02:01,093 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-28T07:02:01,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,094 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,095 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,104 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:01,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:01,105 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636, ASSIGN to state=OFFLINE, location=null, table=test, region=93afe67b3b08d7a0c25e98ea9a0dc636 to restore RIT 2024-11-28T07:02:01,105 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351105 2024-11-28T07:02:01,105 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,105 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,107 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,108 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,108 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,108 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,111 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,113 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,113 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381113 2024-11-28T07:02:01,113 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441113 2024-11-28T07:02:01,113 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-28T07:02:01,113 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,113 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,114 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,114 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,114 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,114 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,114 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,114 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,115 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,116 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,117 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,117 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,117 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,117 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,117 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:01,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,118 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,119 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN 2024-11-28T07:02:01,119 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-28T07:02:01,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,120 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,120 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:01,154 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8db2bf432be6aeda43c73169f2d030a4, ASSIGN to state=OFFLINE, location=null, table=test, region=8db2bf432be6aeda43c73169f2d030a4 to restore RIT 2024-11-28T07:02:01,154 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351154 2024-11-28T07:02:01,155 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,155 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,156 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,157 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,157 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,163 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381163 2024-11-28T07:02:01,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441163 2024-11-28T07:02:01,163 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-28T07:02:01,163 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,163 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,163 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,163 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,164 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,164 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,164 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,164 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,165 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,167 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:01,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,169 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,170 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,171 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,171 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,171 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,171 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,171 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN 2024-11-28T07:02:01,171 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-28T07:02:01,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,172 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,172 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,188 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fbb63e696a2f079596ba1efcff5d041b, ASSIGN to state=OFFLINE, location=null, table=test, region=fbb63e696a2f079596ba1efcff5d041b to restore RIT 2024-11-28T07:02:01,188 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351188 2024-11-28T07:02:01,188 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,188 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,190 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,191 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,191 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,191 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,196 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,196 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381196 2024-11-28T07:02:01,196 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441196 2024-11-28T07:02:01,196 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-28T07:02:01,196 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,196 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,196 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,196 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,196 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,196 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,196 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,197 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:01,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,203 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,203 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-28T07:02:01,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,204 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,205 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351213 2024-11-28T07:02:01,213 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,213 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,215 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,216 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,216 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,216 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,221 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381221 2024-11-28T07:02:01,221 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441221 2024-11-28T07:02:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-28T07:02:01,221 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,221 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,221 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,221 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,221 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,221 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,221 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,221 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,222 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:01,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,225 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,226 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.1732777314425, worker=1cb9cf437a36,44721,1732777312875 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-28T07:02:01,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,226 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,227 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,237 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,237 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,237 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351238 2024-11-28T07:02:01,239 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,239 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,240 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,241 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,241 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,241 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,246 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,246 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381246 2024-11-28T07:02:01,246 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441246 2024-11-28T07:02:01,246 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-28T07:02:01,246 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,246 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,246 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,246 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,246 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,246 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,246 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,247 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,248 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,249 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-28T07:02:01,249 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,250 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,251 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-28T07:02:01,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,252 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,252 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,262 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,262 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,264 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351263 2024-11-28T07:02:01,264 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,264 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,265 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,267 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,267 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,267 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,270 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,272 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,272 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381272 2024-11-28T07:02:01,272 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441272 2024-11-28T07:02:01,272 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-28T07:02:01,272 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,272 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,272 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,272 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,272 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,272 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,272 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,273 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,273 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,275 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,275 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,275 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,275 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-28T07:02:01,275 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,277 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:02:01,277 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,277 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,277 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,277 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,277 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-28T07:02:01,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,278 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,278 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,287 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,288 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351288 2024-11-28T07:02:01,288 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,288 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,290 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,294 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,294 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,294 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,296 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,296 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381296 2024-11-28T07:02:01,296 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441296 2024-11-28T07:02:01,296 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-28T07:02:01,296 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,296 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,296 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,297 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,297 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,297 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,297 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,297 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,298 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-28T07:02:01,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,301 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-28T07:02:01,302 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,302 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,302 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,312 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,314 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351314 2024-11-28T07:02:01,314 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,314 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,315 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,320 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,320 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,320 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,323 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,323 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381323 2024-11-28T07:02:01,323 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441323 2024-11-28T07:02:01,323 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-28T07:02:01,323 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,323 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,323 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,323 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,324 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,324 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,324 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,325 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-28T07:02:01,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,329 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,329 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 1cb9cf437a36%2C42757%2C1732777312688.meta.1732777314857.meta, worker=1cb9cf437a36,38403,1732777312808 didn't hold the lock before restarting, skip acquiring lock. 2024-11-28T07:02:01,329 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-28T07:02:01,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,330 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,330 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,343 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351343 2024-11-28T07:02:01,343 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,343 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,344 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-28T07:02:01,345 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,346 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,349 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,350 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,351 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,353 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,353 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381353 2024-11-28T07:02:01,353 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441353 2024-11-28T07:02:01,353 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-28T07:02:01,353 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-28T07:02:01,353 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,353 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,353 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,353 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-28T07:02:01,354 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-28T07:02:01,354 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-28T07:02:01,354 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:02:01,355 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:02:01,357 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-28T07:02:01,357 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:02:01,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-28T07:02:01,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fa65e80f1531c2a3d21843d8e252160f, REOPEN/MOVE 2024-11-28T07:02:01,358 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true 2024-11-28T07:02:01,358 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-28T07:02:01,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-28T07:02:01,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=65 2024-11-28T07:02:01,360 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=1cb9cf437a36,38403,1732777312808, table=hbase:meta, region=1588230740 2024-11-28T07:02:01,360 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,38403,1732777312808, state=OPEN 2024-11-28T07:02:01,365 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,366 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:02:01,370 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,370 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,370 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:02:01,371 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777351371 2024-11-28T07:02:01,371 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:02:01,371 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:02:01,373 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-28T07:02:01,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=86c81a2b73bbd41aa0789229142fc551, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4276658fc73709d30ba28565a9fbbf03, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5a06150eff3d66689f377c8cac0f8ad, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32789c56ce6b6f4105a05768e5d8e631, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 93afe67b3b08d7a0c25e98ea9a0dc636, server=1cb9cf437a36,44721,1732777312875:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 1cb9cf437a36,42757,1732777312688, splitWal=true, meta=true exec-time=2.5210 sec 2024-11-28T07:02:01,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93afe67b3b08d7a0c25e98ea9a0dc636, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64c03d436be892d03f4313bba416e19f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,376 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3d6a3aee8d008ffd9b93917e3b77bf62, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f41522ba5ab99eefcd7eeb6697805184, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=71bd07899358e3652d4466cdb1dd3176, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c86fc11a5bf22fc0b32716e9c44823d3, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fbb63e696a2f079596ba1efcff5d041b, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b32dc238678e85db1609526ddf98ddb1, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2857a2dc2b09c2ea3bb5615768b39d67, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa7f0c541fab82c2773800078745e787, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89adcb353365cff3d8866fb113c54483, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6b0e706da5bc6b972222c767b7eb31f6, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fa65e80f1531c2a3d21843d8e252160f, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=5 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1471a7fac83428eb937e98338e4948d6, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46d26e2a6997699edcc98e759a48e66a, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8db2bf432be6aeda43c73169f2d030a4, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24cb8e2edb958f7d862266464b772c73, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d610dd7e8db3ac925480924e91da676, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5480f78d30eedd25c01f5f1822c0d5d1, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b6744df8a48cf582efed97c8fb9152b5, regionState=OPEN, lastHost=1cb9cf437a36,44721,1732777312875, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4c79f2e3a60088e6bfee9be7f58096e9, regionState=OPEN, lastHost=1cb9cf437a36,38403,1732777312808, regionLocation=1cb9cf437a36,38403,1732777312808, openSeqNum=2 2024-11-28T07:02:01,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0519f109d08233033823877d6faba4e1, regionState=OPENING, lastHost=1cb9cf437a36,42757,1732777312688, regionLocation=1cb9cf437a36,44721,1732777312875, openSeqNum=2 2024-11-28T07:02:01,378 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:02:01,378 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777381378 2024-11-28T07:02:01,378 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777441378 2024-11-28T07:02:01,378 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-28T07:02:01,397 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=348 (was 381), OpenFileDescriptor=620 (was 642), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=122 (was 122), ProcessCount=11 (was 11), AvailableMemoryMB=8086 (was 8141) 2024-11-28T07:02:01,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T07:02:01,398 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:02:01,398 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:02:01,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:02:01,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:02:01,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:02:01,399 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T07:02:01,399 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=37152252, stopped=false 2024-11-28T07:02:01,399 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1cb9cf437a36,40243,1732777311941 2024-11-28T07:02:01,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:02:01,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:02:01,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:02:01,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:02:01,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:02:01,409 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:02:01,409 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:02:01,409 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:02:01,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:02:01,409 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:02:01,409 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:02:01,410 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:02:01,410 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1cb9cf437a36,38403,1732777312808' ***** 2024-11-28T07:02:01,410 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T07:02:01,410 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1cb9cf437a36,44721,1732777312875' ***** 2024-11-28T07:02:01,410 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T07:02:01,410 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T07:02:01,410 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T07:02:01,410 INFO [RS:1;1cb9cf437a36:38403 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T07:02:01,410 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T07:02:01,410 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T07:02:01,410 INFO [RS:1;1cb9cf437a36:38403 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T07:02:01,410 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 86c81a2b73bbd41aa0789229142fc551 2024-11-28T07:02:01,410 INFO [RS:2;1cb9cf437a36:44721 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T07:02:01,410 INFO [RS:2;1cb9cf437a36:44721 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T07:02:01,410 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for 24cb8e2edb958f7d862266464b772c73 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 5480f78d30eedd25c01f5f1822c0d5d1 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 4d610dd7e8db3ac925480924e91da676 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 46d26e2a6997699edcc98e759a48e66a 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 32789c56ce6b6f4105a05768e5d8e631 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 6b0e706da5bc6b972222c767b7eb31f6 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for f41522ba5ab99eefcd7eeb6697805184 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 4c79f2e3a60088e6bfee9be7f58096e9 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 2857a2dc2b09c2ea3bb5615768b39d67 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 3d6a3aee8d008ffd9b93917e3b77bf62 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for c86fc11a5bf22fc0b32716e9c44823d3 2024-11-28T07:02:01,411 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 86c81a2b73bbd41aa0789229142fc551, disabling compactions & flushes 2024-11-28T07:02:01,411 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(959): stopping server 1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,411 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:02:01,411 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. after waiting 0 ms 2024-11-28T07:02:01,411 INFO [RS:1;1cb9cf437a36:38403 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1cb9cf437a36:38403. 2024-11-28T07:02:01,412 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:02:01,412 DEBUG [RS:1;1cb9cf437a36:38403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:02:01,412 DEBUG [RS:1;1cb9cf437a36:38403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:02:01,412 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T07:02:01,412 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T07:02:01,412 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T07:02:01,412 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T07:02:01,412 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:02:01,412 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for 64c03d436be892d03f4313bba416e19f 2024-11-28T07:02:01,412 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for b32dc238678e85db1609526ddf98ddb1 2024-11-28T07:02:01,412 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 24cb8e2edb958f7d862266464b772c73, disabling compactions & flushes 2024-11-28T07:02:01,412 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for 71bd07899358e3652d4466cdb1dd3176 2024-11-28T07:02:01,412 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for b6744df8a48cf582efed97c8fb9152b5 2024-11-28T07:02:01,412 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:02:01,412 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for 4276658fc73709d30ba28565a9fbbf03 2024-11-28T07:02:01,412 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:02:01,412 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. after waiting 0 ms 2024-11-28T07:02:01,412 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for 93afe67b3b08d7a0c25e98ea9a0dc636 2024-11-28T07:02:01,412 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:02:01,413 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for 1471a7fac83428eb937e98338e4948d6 2024-11-28T07:02:01,413 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for 89adcb353365cff3d8866fb113c54483 2024-11-28T07:02:01,413 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(3091): Received CLOSE for 8db2bf432be6aeda43c73169f2d030a4 2024-11-28T07:02:01,413 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(959): stopping server 1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,413 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:02:01,413 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-28T07:02:01,413 INFO [RS:2;1cb9cf437a36:44721 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1cb9cf437a36:44721. 2024-11-28T07:02:01,413 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T07:02:01,413 DEBUG [RS:2;1cb9cf437a36:44721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:02:01,413 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T07:02:01,413 DEBUG [RS:2;1cb9cf437a36:44721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:02:01,413 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T07:02:01,413 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T07:02:01,413 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-28T07:02:01,413 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T07:02:01,413 DEBUG [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2857a2dc2b09c2ea3bb5615768b39d67, 32789c56ce6b6f4105a05768e5d8e631, 3d6a3aee8d008ffd9b93917e3b77bf62, 46d26e2a6997699edcc98e759a48e66a, 4c79f2e3a60088e6bfee9be7f58096e9, 4d610dd7e8db3ac925480924e91da676, 5480f78d30eedd25c01f5f1822c0d5d1, 6b0e706da5bc6b972222c767b7eb31f6, 86c81a2b73bbd41aa0789229142fc551, c86fc11a5bf22fc0b32716e9c44823d3, f41522ba5ab99eefcd7eeb6697805184, fa7f0c541fab82c2773800078745e787 2024-11-28T07:02:01,413 DEBUG [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(1351): Waiting on 0519f109d08233033823877d6faba4e1, 1471a7fac83428eb937e98338e4948d6, 24cb8e2edb958f7d862266464b772c73, 4276658fc73709d30ba28565a9fbbf03, 64c03d436be892d03f4313bba416e19f, 71bd07899358e3652d4466cdb1dd3176, 89adcb353365cff3d8866fb113c54483, 8db2bf432be6aeda43c73169f2d030a4, 93afe67b3b08d7a0c25e98ea9a0dc636, b32dc238678e85db1609526ddf98ddb1, b6744df8a48cf582efed97c8fb9152b5, fa65e80f1531c2a3d21843d8e252160f 2024-11-28T07:02:01,413 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-11-28T07:02:01,418 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/86c81a2b73bbd41aa0789229142fc551/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,419 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/24cb8e2edb958f7d862266464b772c73/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,419 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:02:01,419 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 86c81a2b73bbd41aa0789229142fc551: Waiting for close lock at 1732777321411Running coprocessor pre-close hooks at 1732777321411Disabling compacts and flushes for region at 1732777321411Disabling writes for close at 1732777321411Writing region close event to WAL at 1732777321412 (+1 ms)Running coprocessor post-close hooks at 1732777321419 (+7 ms)Closed at 1732777321419 2024-11-28T07:02:01,419 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732777315307.86c81a2b73bbd41aa0789229142fc551. 2024-11-28T07:02:01,419 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5480f78d30eedd25c01f5f1822c0d5d1, disabling compactions & flushes 2024-11-28T07:02:01,419 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:02:01,419 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:02:01,420 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. after waiting 0 ms 2024-11-28T07:02:01,420 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:02:01,420 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:02:01,420 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 24cb8e2edb958f7d862266464b772c73: Waiting for close lock at 1732777321412Running coprocessor pre-close hooks at 1732777321412Disabling compacts and flushes for region at 1732777321412Disabling writes for close at 1732777321412Writing region close event to WAL at 1732777321415 (+3 ms)Running coprocessor post-close hooks at 1732777321419 (+4 ms)Closed at 1732777321420 (+1 ms) 2024-11-28T07:02:01,420 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732777315307.24cb8e2edb958f7d862266464b772c73. 2024-11-28T07:02:01,420 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fa65e80f1531c2a3d21843d8e252160f, disabling compactions & flushes 2024-11-28T07:02:01,420 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:02:01,420 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:02:01,420 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. after waiting 0 ms 2024-11-28T07:02:01,420 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:02:01,424 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/5480f78d30eedd25c01f5f1822c0d5d1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,425 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/fa65e80f1531c2a3d21843d8e252160f/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-28T07:02:01,425 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:02:01,425 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5480f78d30eedd25c01f5f1822c0d5d1: Waiting for close lock at 1732777321419Running coprocessor pre-close hooks at 1732777321419Disabling compacts and flushes for region at 1732777321419Disabling writes for close at 1732777321420 (+1 ms)Writing region close event to WAL at 1732777321420Running coprocessor post-close hooks at 1732777321425 (+5 ms)Closed at 1732777321425 2024-11-28T07:02:01,425 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732777315307.5480f78d30eedd25c01f5f1822c0d5d1. 2024-11-28T07:02:01,425 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:02:01,425 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4d610dd7e8db3ac925480924e91da676, disabling compactions & flushes 2024-11-28T07:02:01,425 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:02:01,425 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fa65e80f1531c2a3d21843d8e252160f: Waiting for close lock at 1732777321420Running coprocessor pre-close hooks at 1732777321420Disabling compacts and flushes for region at 1732777321420Disabling writes for close at 1732777321420Writing region close event to WAL at 1732777321421 (+1 ms)Running coprocessor post-close hooks at 1732777321425 (+4 ms)Closed at 1732777321425 2024-11-28T07:02:01,425 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:02:01,426 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. after waiting 0 ms 2024-11-28T07:02:01,426 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732777315307.fa65e80f1531c2a3d21843d8e252160f. 2024-11-28T07:02:01,426 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:02:01,426 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 64c03d436be892d03f4313bba416e19f, disabling compactions & flushes 2024-11-28T07:02:01,426 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:02:01,426 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:02:01,426 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. after waiting 0 ms 2024-11-28T07:02:01,426 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:02:01,426 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/info/8e247bf96bd641ee99782d90a216b05b is 138, key is test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad./info:regioninfo/1732777320249/Put/seqid=0 2024-11-28T07:02:01,431 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/64c03d436be892d03f4313bba416e19f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,432 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:02:01,432 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 64c03d436be892d03f4313bba416e19f: Waiting for close lock at 1732777321426Running coprocessor pre-close hooks at 1732777321426Disabling compacts and flushes for region at 1732777321426Disabling writes for close at 1732777321426Writing region close event to WAL at 1732777321427 (+1 ms)Running coprocessor post-close hooks at 1732777321432 (+5 ms)Closed at 1732777321432 2024-11-28T07:02:01,432 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732777315307.64c03d436be892d03f4313bba416e19f. 2024-11-28T07:02:01,432 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b32dc238678e85db1609526ddf98ddb1, disabling compactions & flushes 2024-11-28T07:02:01,432 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:02:01,432 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:02:01,433 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. after waiting 0 ms 2024-11-28T07:02:01,433 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:02:01,435 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4d610dd7e8db3ac925480924e91da676/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,436 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:02:01,436 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4d610dd7e8db3ac925480924e91da676: Waiting for close lock at 1732777321425Running coprocessor pre-close hooks at 1732777321425Disabling compacts and flushes for region at 1732777321425Disabling writes for close at 1732777321426 (+1 ms)Writing region close event to WAL at 1732777321427 (+1 ms)Running coprocessor post-close hooks at 1732777321436 (+9 ms)Closed at 1732777321436 2024-11-28T07:02:01,436 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732777315307.4d610dd7e8db3ac925480924e91da676. 2024-11-28T07:02:01,437 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46d26e2a6997699edcc98e759a48e66a, disabling compactions & flushes 2024-11-28T07:02:01,437 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:02:01,437 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:02:01,437 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. after waiting 0 ms 2024-11-28T07:02:01,437 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:02:01,447 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/b32dc238678e85db1609526ddf98ddb1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,447 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/46d26e2a6997699edcc98e759a48e66a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,448 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:02:01,448 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b32dc238678e85db1609526ddf98ddb1: Waiting for close lock at 1732777321432Running coprocessor pre-close hooks at 1732777321432Disabling compacts and flushes for region at 1732777321432Disabling writes for close at 1732777321433 (+1 ms)Writing region close event to WAL at 1732777321439 (+6 ms)Running coprocessor post-close hooks at 1732777321448 (+9 ms)Closed at 1732777321448 2024-11-28T07:02:01,448 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732777315307.b32dc238678e85db1609526ddf98ddb1. 2024-11-28T07:02:01,448 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:02:01,449 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46d26e2a6997699edcc98e759a48e66a: Waiting for close lock at 1732777321437Running coprocessor pre-close hooks at 1732777321437Disabling compacts and flushes for region at 1732777321437Disabling writes for close at 1732777321437Writing region close event to WAL at 1732777321439 (+2 ms)Running coprocessor post-close hooks at 1732777321448 (+9 ms)Closed at 1732777321448 2024-11-28T07:02:01,449 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 71bd07899358e3652d4466cdb1dd3176, disabling compactions & flushes 2024-11-28T07:02:01,449 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732777315307.46d26e2a6997699edcc98e759a48e66a. 2024-11-28T07:02:01,449 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 32789c56ce6b6f4105a05768e5d8e631, disabling compactions & flushes 2024-11-28T07:02:01,449 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:02:01,449 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:02:01,449 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. after waiting 0 ms 2024-11-28T07:02:01,449 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:02:01,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741869_1045 (size=8192) 2024-11-28T07:02:01,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741869_1045 (size=8192) 2024-11-28T07:02:01,449 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:02:01,449 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:02:01,450 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. after waiting 0 ms 2024-11-28T07:02:01,450 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:02:01,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741869_1045 (size=8192) 2024-11-28T07:02:01,451 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/info/8e247bf96bd641ee99782d90a216b05b 2024-11-28T07:02:01,454 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/32789c56ce6b6f4105a05768e5d8e631/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,455 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/71bd07899358e3652d4466cdb1dd3176/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,455 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:02:01,455 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 32789c56ce6b6f4105a05768e5d8e631: Waiting for close lock at 1732777321449Running coprocessor pre-close hooks at 1732777321449Disabling compacts and flushes for region at 1732777321449Disabling writes for close at 1732777321449Writing region close event to WAL at 1732777321450 (+1 ms)Running coprocessor post-close hooks at 1732777321455 (+5 ms)Closed at 1732777321455 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732777315307.32789c56ce6b6f4105a05768e5d8e631. 2024-11-28T07:02:01,456 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6b0e706da5bc6b972222c767b7eb31f6, disabling compactions & flushes 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 71bd07899358e3652d4466cdb1dd3176: Waiting for close lock at 1732777321449Running coprocessor pre-close hooks at 1732777321449Disabling compacts and flushes for region at 1732777321449Disabling writes for close at 1732777321450 (+1 ms)Writing region close event to WAL at 1732777321450Running coprocessor post-close hooks at 1732777321456 (+6 ms)Closed at 1732777321456 2024-11-28T07:02:01,456 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732777315307.71bd07899358e3652d4466cdb1dd3176. 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. after waiting 0 ms 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b6744df8a48cf582efed97c8fb9152b5, disabling compactions & flushes 2024-11-28T07:02:01,456 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. after waiting 0 ms 2024-11-28T07:02:01,456 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:02:01,462 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/.tmp/info/8e247bf96bd641ee99782d90a216b05b as hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/info/8e247bf96bd641ee99782d90a216b05b 2024-11-28T07:02:01,462 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/b6744df8a48cf582efed97c8fb9152b5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,462 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/6b0e706da5bc6b972222c767b7eb31f6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,463 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:02:01,463 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6b0e706da5bc6b972222c767b7eb31f6: Waiting for close lock at 1732777321456Running coprocessor pre-close hooks at 1732777321456Disabling compacts and flushes for region at 1732777321456Disabling writes for close at 1732777321456Writing region close event to WAL at 1732777321457 (+1 ms)Running coprocessor post-close hooks at 1732777321463 (+6 ms)Closed at 1732777321463 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b6744df8a48cf582efed97c8fb9152b5: Waiting for close lock at 1732777321456Running coprocessor pre-close hooks at 1732777321456Disabling compacts and flushes for region at 1732777321456Disabling writes for close at 1732777321456Writing region close event to WAL at 1732777321457 (+1 ms)Running coprocessor post-close hooks at 1732777321463 (+6 ms)Closed at 1732777321463 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732777315307.6b0e706da5bc6b972222c767b7eb31f6. 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732777315307.b6744df8a48cf582efed97c8fb9152b5. 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f41522ba5ab99eefcd7eeb6697805184, disabling compactions & flushes 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4276658fc73709d30ba28565a9fbbf03, disabling compactions & flushes 2024-11-28T07:02:01,463 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:02:01,463 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. after waiting 0 ms 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. after waiting 0 ms 2024-11-28T07:02:01,463 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:02:01,468 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4276658fc73709d30ba28565a9fbbf03/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,468 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/f41522ba5ab99eefcd7eeb6697805184/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,469 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:02:01,469 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4276658fc73709d30ba28565a9fbbf03: Waiting for close lock at 1732777321463Running coprocessor pre-close hooks at 1732777321463Disabling compacts and flushes for region at 1732777321463Disabling writes for close at 1732777321463Writing region close event to WAL at 1732777321464 (+1 ms)Running coprocessor post-close hooks at 1732777321469 (+5 ms)Closed at 1732777321469 2024-11-28T07:02:01,469 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:02:01,469 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f41522ba5ab99eefcd7eeb6697805184: Waiting for close lock at 1732777321463Running coprocessor pre-close hooks at 1732777321463Disabling compacts and flushes for region at 1732777321463Disabling writes for close at 1732777321463Writing region close event to WAL at 1732777321464 (+1 ms)Running coprocessor post-close hooks at 1732777321469 (+5 ms)Closed at 1732777321469 2024-11-28T07:02:01,469 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732777315307.4276658fc73709d30ba28565a9fbbf03. 2024-11-28T07:02:01,469 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732777315307.f41522ba5ab99eefcd7eeb6697805184. 2024-11-28T07:02:01,469 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4c79f2e3a60088e6bfee9be7f58096e9, disabling compactions & flushes 2024-11-28T07:02:01,469 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 93afe67b3b08d7a0c25e98ea9a0dc636, disabling compactions & flushes 2024-11-28T07:02:01,469 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:02:01,469 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:02:01,469 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:02:01,469 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:02:01,469 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. after waiting 0 ms 2024-11-28T07:02:01,470 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. after waiting 0 ms 2024-11-28T07:02:01,470 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:02:01,470 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:02:01,471 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/info/8e247bf96bd641ee99782d90a216b05b, entries=24, sequenceid=75, filesize=8 K 2024-11-28T07:02:01,472 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 59ms, sequenceid=75, compaction requested=false 2024-11-28T07:02:01,475 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/4c79f2e3a60088e6bfee9be7f58096e9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,475 INFO [regionserver/1cb9cf437a36:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:02:01,475 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:02:01,476 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4c79f2e3a60088e6bfee9be7f58096e9: Waiting for close lock at 1732777321469Running coprocessor pre-close hooks at 1732777321469Disabling compacts and flushes for region at 1732777321469Disabling writes for close at 1732777321470 (+1 ms)Writing region close event to WAL at 1732777321470Running coprocessor post-close hooks at 1732777321475 (+5 ms)Closed at 1732777321475 2024-11-28T07:02:01,476 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732777315307.4c79f2e3a60088e6bfee9be7f58096e9. 2024-11-28T07:02:01,476 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2857a2dc2b09c2ea3bb5615768b39d67, disabling compactions & flushes 2024-11-28T07:02:01,476 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:02:01,476 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:02:01,476 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. after waiting 0 ms 2024-11-28T07:02:01,476 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:02:01,476 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/93afe67b3b08d7a0c25e98ea9a0dc636/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,477 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:02:01,478 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 93afe67b3b08d7a0c25e98ea9a0dc636: Waiting for close lock at 1732777321469Running coprocessor pre-close hooks at 1732777321469Disabling compacts and flushes for region at 1732777321469Disabling writes for close at 1732777321470 (+1 ms)Writing region close event to WAL at 1732777321470Running coprocessor post-close hooks at 1732777321477 (+7 ms)Closed at 1732777321477 2024-11-28T07:02:01,478 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732777315307.93afe67b3b08d7a0c25e98ea9a0dc636. 2024-11-28T07:02:01,478 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1471a7fac83428eb937e98338e4948d6, disabling compactions & flushes 2024-11-28T07:02:01,478 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:02:01,478 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:02:01,478 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. after waiting 0 ms 2024-11-28T07:02:01,478 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:02:01,479 INFO [regionserver/1cb9cf437a36:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:02:01,479 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-28T07:02:01,479 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T07:02:01,480 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T07:02:01,480 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732777321413Running coprocessor pre-close hooks at 1732777321413Disabling compacts and flushes for region at 1732777321413Disabling writes for close at 1732777321413Obtaining lock to block concurrent updates at 1732777321413Preparing flush snapshotting stores in 1588230740 at 1732777321413Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732777321420 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732777321422 (+2 ms)Flushing 1588230740/info: creating writer at 1732777321422Flushing 1588230740/info: appending metadata at 1732777321426 (+4 ms)Flushing 1588230740/info: closing flushed file at 1732777321426Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e35d77f: reopening flushed file at 1732777321461 (+35 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 59ms, sequenceid=75, compaction requested=false at 1732777321472 (+11 ms)Writing region close event to WAL at 1732777321475 (+3 ms)Running coprocessor post-close hooks at 1732777321479 (+4 ms)Closed at 1732777321480 (+1 ms) 2024-11-28T07:02:01,480 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T07:02:01,482 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/2857a2dc2b09c2ea3bb5615768b39d67/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,482 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/1471a7fac83428eb937e98338e4948d6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,482 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:02:01,483 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1471a7fac83428eb937e98338e4948d6: Waiting for close lock at 1732777321478Running coprocessor pre-close hooks at 1732777321478Disabling compacts and flushes for region at 1732777321478Disabling writes for close at 1732777321478Writing region close event to WAL at 1732777321478Running coprocessor post-close hooks at 1732777321482 (+4 ms)Closed at 1732777321482 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2857a2dc2b09c2ea3bb5615768b39d67: Waiting for close lock at 1732777321476Running coprocessor pre-close hooks at 1732777321476Disabling compacts and flushes for region at 1732777321476Disabling writes for close at 1732777321476Writing region close event to WAL at 1732777321477 (+1 ms)Running coprocessor post-close hooks at 1732777321482 (+5 ms)Closed at 1732777321482 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732777315307.1471a7fac83428eb937e98338e4948d6. 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732777315307.2857a2dc2b09c2ea3bb5615768b39d67. 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 89adcb353365cff3d8866fb113c54483, disabling compactions & flushes 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3d6a3aee8d008ffd9b93917e3b77bf62, disabling compactions & flushes 2024-11-28T07:02:01,483 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:02:01,483 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. after waiting 0 ms 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. after waiting 0 ms 2024-11-28T07:02:01,483 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:02:01,487 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/89adcb353365cff3d8866fb113c54483/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,488 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:02:01,488 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 89adcb353365cff3d8866fb113c54483: Waiting for close lock at 1732777321483Running coprocessor pre-close hooks at 1732777321483Disabling compacts and flushes for region at 1732777321483Disabling writes for close at 1732777321483Writing region close event to WAL at 1732777321483Running coprocessor post-close hooks at 1732777321488 (+5 ms)Closed at 1732777321488 2024-11-28T07:02:01,488 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732777315307.89adcb353365cff3d8866fb113c54483. 2024-11-28T07:02:01,488 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8db2bf432be6aeda43c73169f2d030a4, disabling compactions & flushes 2024-11-28T07:02:01,488 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:02:01,488 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:02:01,488 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. after waiting 0 ms 2024-11-28T07:02:01,488 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:02:01,489 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/3d6a3aee8d008ffd9b93917e3b77bf62/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,489 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:02:01,490 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3d6a3aee8d008ffd9b93917e3b77bf62: Waiting for close lock at 1732777321483Running coprocessor pre-close hooks at 1732777321483Disabling compacts and flushes for region at 1732777321483Disabling writes for close at 1732777321483Writing region close event to WAL at 1732777321485 (+2 ms)Running coprocessor post-close hooks at 1732777321489 (+4 ms)Closed at 1732777321489 2024-11-28T07:02:01,490 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732777315307.3d6a3aee8d008ffd9b93917e3b77bf62. 2024-11-28T07:02:01,490 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c86fc11a5bf22fc0b32716e9c44823d3, disabling compactions & flushes 2024-11-28T07:02:01,490 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:02:01,490 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:02:01,490 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. after waiting 0 ms 2024-11-28T07:02:01,490 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:02:01,493 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/8db2bf432be6aeda43c73169f2d030a4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,493 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:02:01,493 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8db2bf432be6aeda43c73169f2d030a4: Waiting for close lock at 1732777321488Running coprocessor pre-close hooks at 1732777321488Disabling compacts and flushes for region at 1732777321488Disabling writes for close at 1732777321488Writing region close event to WAL at 1732777321489 (+1 ms)Running coprocessor post-close hooks at 1732777321493 (+4 ms)Closed at 1732777321493 2024-11-28T07:02:01,493 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732777315307.8db2bf432be6aeda43c73169f2d030a4. 2024-11-28T07:02:01,494 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/data/default/test/c86fc11a5bf22fc0b32716e9c44823d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:02:01,495 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:02:01,495 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c86fc11a5bf22fc0b32716e9c44823d3: Waiting for close lock at 1732777321490Running coprocessor pre-close hooks at 1732777321490Disabling compacts and flushes for region at 1732777321490Disabling writes for close at 1732777321490Writing region close event to WAL at 1732777321491 (+1 ms)Running coprocessor post-close hooks at 1732777321495 (+4 ms)Closed at 1732777321495 2024-11-28T07:02:01,495 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732777315307.c86fc11a5bf22fc0b32716e9c44823d3. 2024-11-28T07:02:01,599 WARN [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787., aborting... java.io.IOException: Failed to report opened region to master: test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:01,600 ERROR [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server 1cb9cf437a36,38403,1732777312808: Failed to open region test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,nnn,1732777315307.fa7f0c541fab82c2773800078745e787. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:01,600 ERROR [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-28T07:02:01,600 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-28T07:02:01,601 WARN [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,zzz,1732777315307.0519f109d08233033823877d6faba4e1., aborting... java.io.IOException: Failed to report opened region to master: test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:01,602 ERROR [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2440): ***** ABORTING region server 1cb9cf437a36,44721,1732777312875: Failed to open region test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,zzz,1732777315307.0519f109d08233033823877d6faba4e1. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:01,602 ERROR [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-28T07:02:01,603 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-28T07:02:01,603 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-28T07:02:01,604 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-28T07:02:01,610 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-28T07:02:01,611 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-28T07:02:01,611 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-28T07:02:01,612 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-28T07:02:01,613 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.HRegionServer(976): stopping server 1cb9cf437a36,38403,1732777312808; all regions closed. 2024-11-28T07:02:01,613 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(976): stopping server 1cb9cf437a36,44721,1732777312875; all regions closed. 2024-11-28T07:02:01,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741865_1041 (size=4201) 2024-11-28T07:02:01,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741833_1009 (size=6268) 2024-11-28T07:02:01,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741865_1041 (size=4201) 2024-11-28T07:02:01,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741833_1009 (size=6268) 2024-11-28T07:02:01,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741833_1009 (size=6268) 2024-11-28T07:02:01,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741865_1041 (size=4201) 2024-11-28T07:02:01,618 DEBUG [RS:2;1cb9cf437a36:44721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:02:01,618 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:02:01,618 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:02:01,618 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.ChoreService(370): Chore service for: regionserver/1cb9cf437a36:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T07:02:01,618 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T07:02:01,618 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T07:02:01,618 INFO [regionserver/1cb9cf437a36:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:02:01,619 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T07:02:01,619 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:02:01,620 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 508242496 }, "NonHeapMemoryUsage": { "committed": 163971072, "init": 7667712, "max": -1, "used": 161363208 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "1cb9cf437a36", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 38606, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 113, "ProcessCallTime_min": 0, "ProcessCallTime_max": 17, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 8, "ProcessCallTime_75th_percentile": 12, "ProcessCallTime_90th_percentile": 15, "ProcessCallTime_95th_percentile": 16, "ProcessCallTime_98th_percentile": 16, "ProcessCallTime_99th_percentile": 16, "ProcessCallTime_99.9th_percentile": 16, "ProcessCallTime_TimeRangeCount_0-1": 113, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 113, "QueueCallTime_min": 0, "QueueCallTime_max": 11, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 8, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 10, "QueueCallTime_98th_percentile": 10, "QueueCallTime_99th_percentile": 10, "QueueCallTime_99.9th_percentile": 10, "QueueCallTime_TimeRangeCount_0-1": 113, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 113, "TotalCallTime_min": 0, "TotalCallTime_max": 17, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 8, "TotalCallTime_75th_percentile": 12, "TotalCallTime_90th_percentile": 15, "TotalCallTime_95th_percentile": 16, "TotalCallTime_98th_percentile": 16, "TotalCallTime_99th_percentile": 16, "TotalCallTime_99.9th_percentile": 16, "TotalCallTime_TimeRangeCount_0-1": 113, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 113, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 605, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 113, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 113, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 309, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 113, "sentBytes": 54868 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "1cb9cf437a36", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49831", "tag.serverName": "1cb9cf437a36,38403,1732777312808", "tag.clusterId": "15a2db50-5199-4be0-a816-ae6d383659bb", "tag.Context": "regionserver", "tag.Hostname": "1cb9cf437a36", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732777312808, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3327, "localBytesRead": 3327, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7446, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 17, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 0, "Put_max": 1, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46689, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46689, "FlushOutputSize_max": 46689, "FlushOutputSize_mean": 46689, "FlushOutputSize_25th_percentile": 46689, "FlushOutputSize_median": 46689, "FlushOutputSize_75th_percentile": 46689, "FlushOutputSize_90th_percentile": 46689, "FlushOutputSize_95th_percentile": 46689, "FlushOutputSize_98th_percentile": 46689, "FlushOutputSize_99th_percentile": 46689, "FlushOutputSize_99.9th_percentile": 46689, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 231, "FlushTime_max": 231, "FlushTime_mean": 231, "FlushTime_25th_percentile": 231, "FlushTime_median": 231, "FlushTime_75th_percentile": 231, "FlushTime_90th_percentile": 231, "FlushTime_95th_percentile": 231, "FlushTime_98th_percentile": 231, "FlushTime_99th_percentile": 231, "FlushTime_99.9th_percentile": 231, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-28T07:02:01,621 INFO [RS:2;1cb9cf437a36:44721 {}] executor.ExecutorService(108): Executor-9-RS_OPEN_REGION-regionserver/1cb9cf437a36:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-28T07:02:01,621 INFO [RS:2;1cb9cf437a36:44721 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44721 2024-11-28T07:02:01,622 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 508242496 }, "NonHeapMemoryUsage": { "committed": 163971072, "init": 7667712, "max": -1, "used": 161359768 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "1cb9cf437a36", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 38606, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 113, "ProcessCallTime_min": 0, "ProcessCallTime_max": 17, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 8, "ProcessCallTime_75th_percentile": 12, "ProcessCallTime_90th_percentile": 15, "ProcessCallTime_95th_percentile": 16, "ProcessCallTime_98th_percentile": 16, "ProcessCallTime_99th_percentile": 16, "ProcessCallTime_99.9th_percentile": 16, "ProcessCallTime_TimeRangeCount_0-1": 113, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 113, "QueueCallTime_min": 0, "QueueCallTime_max": 11, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 2, "QueueCallTime_median": 5, "QueueCallTime_75th_percentile": 8, "QueueCallTime_90th_percentile": 9, "QueueCallTime_95th_percentile": 10, "QueueCallTime_98th_percentile": 10, "QueueCallTime_99th_percentile": 10, "QueueCallTime_99.9th_percentile": 10, "QueueCallTime_TimeRangeCount_0-1": 113, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 113, "TotalCallTime_min": 0, "TotalCallTime_max": 17, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 8, "TotalCallTime_75th_percentile": 12, "TotalCallTime_90th_percentile": 15, "TotalCallTime_95th_percentile": 16, "TotalCallTime_98th_percentile": 16, "TotalCallTime_99th_percentile": 16, "TotalCallTime_99.9th_percentile": 16, "TotalCallTime_TimeRangeCount_0-1": 113, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 113, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 605, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 113, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 113, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 309, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 113, "sentBytes": 54868 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "1cb9cf437a36", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49831", "tag.serverName": "1cb9cf437a36,38403,1732777312808", "tag.clusterId": "15a2db50-5199-4be0-a816-ae6d383659bb", "tag.Context": "regionserver", "tag.Hostname": "1cb9cf437a36", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732777312808, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3327, "localBytesRead": 3327, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7446, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 17, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 8, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 8, "Put_min": 0, "Put_max": 1, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "Put_TimeRangeCount_0-1": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46689, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46689, "FlushOutputSize_max": 46689, "FlushOutputSize_mean": 46689, "FlushOutputSize_25th_percentile": 46689, "FlushOutputSize_median": 46689, "FlushOutputSize_75th_percentile": 46689, "FlushOutputSize_90th_percentile": 46689, "FlushOutputSize_95th_percentile": 46689, "FlushOutputSize_98th_percentile": 46689, "FlushOutputSize_99th_percentile": 46689, "FlushOutputSize_99.9th_percentile": 46689, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 231, "FlushTime_max": 231, "FlushTime_mean": 231, "FlushTime_25th_percentile": 231, "FlushTime_median": 231, "FlushTime_75th_percentile": 231, "FlushTime_90th_percentile": 231, "FlushTime_95th_percentile": 231, "FlushTime_98th_percentile": 231, "FlushTime_99th_percentile": 231, "FlushTime_99.9th_percentile": 231, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-28T07:02:01,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741834_1010 (size=6258) 2024-11-28T07:02:01,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741834_1010 (size=6258) 2024-11-28T07:02:01,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741834_1010 (size=6258) 2024-11-28T07:02:01,623 WARN [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=1cb9cf437a36:40243 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=1cb9cf437a36:40243 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more 2024-11-28T07:02:01,624 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:02:01,624 DEBUG [RS:1;1cb9cf437a36:38403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:02:01,625 INFO [RS:1;1cb9cf437a36:38403 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:02:01,625 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:02:01,625 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => f5a06150eff3d66689f377c8cac0f8ad, NAME => 'test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-28T07:02:01,625 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.ChoreService(370): Chore service for: regionserver/1cb9cf437a36:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T07:02:01,625 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:02:01,625 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:02:01,625 INFO [regionserver/1cb9cf437a36:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:02:01,625 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:02:01,625 INFO [RS:1;1cb9cf437a36:38403 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38403 2024-11-28T07:02:01,625 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:02:01,625 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for f5a06150eff3d66689f377c8cac0f8ad 2024-11-28T07:02:01,626 WARN [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(979): Failed initialize of region= test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad., starting to roll back memstore java.io.InterruptedIOException: Call interrupted at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy41.getFileInfo(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getFileInfo$41(ClientNamenodeProtocolTranslatorPB.java:820) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:820) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy42.getFileInfo(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getFileInfo(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getFileInfo(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1770) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1828) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1825) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1840) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:462) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.checkRegionInfoOnFilesystem(HRegionFileSystem.java:834) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:1027) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:976) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7799) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:01,627 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for f5a06150eff3d66689f377c8cac0f8ad: Running coprocessor pre-open hook at 1732777321626Writing region info on filesystem at 1732777321626Exception during region test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. initialization. at 1732777321627 (+1 ms) 2024-11-28T07:02:01,627 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1722): Closing f5a06150eff3d66689f377c8cac0f8ad, disabling compactions & flushes 2024-11-28T07:02:01,627 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1755): Closing region test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:02:01,627 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. 2024-11-28T07:02:01,627 WARN [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1802): Interrupted while waiting for close lock on test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:01,628 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for f5a06150eff3d66689f377c8cac0f8ad: Waiting for close lock at 1732777321627Running coprocessor pre-close hooks at 1732777321627Disabling compacts and flushes for region at 1732777321627 2024-11-28T07:02:01,628 WARN [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7819): Open region: {ENCODED => f5a06150eff3d66689f377c8cac0f8ad, NAME => 'test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad.', STARTKEY => 'ccc', ENDKEY => 'ddd'} failed. Try close region but got exception java.io.InterruptedIOException: Interrupted while waiting for close lock on test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1803) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1672) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1627) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1610) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7817) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:993) ~[?:?] at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1131) ~[?:?] at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1794) ~[classes/:?] ... 13 more 2024-11-28T07:02:01,628 WARN [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(84): Failed to open region test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad., will report to master java.io.InterruptedIOException: Call interrupted at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy41.getFileInfo(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$getFileInfo$41(ClientNamenodeProtocolTranslatorPB.java:820) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:820) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy42.getFileInfo(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getFileInfo(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy45.getFileInfo(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1770) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1828) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1825) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1840) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:462) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.checkRegionInfoOnFilesystem(HRegionFileSystem.java:834) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:1027) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:976) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7799) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:7754) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7729) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7688) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7644) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:143) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:01,629 WARN [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad., aborting... java.io.IOException: Failed to report failed open to master: test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.cleanUpAndReportFailure(AssignRegionHandler.java:93) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:02:01,629 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,ccc,1732777315307.f5a06150eff3d66689f377c8cac0f8ad. and can not recover 2024-11-28T07:02:01,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1cb9cf437a36,44721,1732777312875 2024-11-28T07:02:01,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:02:01,653 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:02:01,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38403-0x10181890e5a0002, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1cb9cf437a36,38403,1732777312808 2024-11-28T07:02:01,662 INFO [RS:1;1cb9cf437a36:38403 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:02:01,670 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1cb9cf437a36,38403,1732777312808] 2024-11-28T07:02:01,737 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1cb9cf437a36,38403,1732777312808 already deleted, retry=false 2024-11-28T07:02:01,737 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1cb9cf437a36,38403,1732777312808 expired; onlineServers=1 2024-11-28T07:02:01,737 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1cb9cf437a36,44721,1732777312875] 2024-11-28T07:02:01,745 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1cb9cf437a36,44721,1732777312875 already deleted, retry=false 2024-11-28T07:02:01,746 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1cb9cf437a36,44721,1732777312875 expired; onlineServers=0 2024-11-28T07:02:01,746 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1cb9cf437a36,40243,1732777311941' ***** 2024-11-28T07:02:01,746 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T07:02:01,747 INFO [M:0;1cb9cf437a36:40243 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:02:01,747 INFO [M:0;1cb9cf437a36:40243 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:02:01,748 DEBUG [M:0;1cb9cf437a36:40243 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T07:02:01,748 DEBUG [M:0;1cb9cf437a36:40243 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T07:02:01,748 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T07:02:01,748 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732777314063 {}] cleaner.HFileCleaner(306): Exit Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732777314063,5,FailOnTimeoutGroup] 2024-11-28T07:02:01,748 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732777314060 {}] cleaner.HFileCleaner(306): Exit Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732777314060,5,FailOnTimeoutGroup] 2024-11-28T07:02:01,749 INFO [M:0;1cb9cf437a36:40243 {}] hbase.ChoreService(370): Chore service for: master/1cb9cf437a36:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T07:02:01,749 INFO [M:0;1cb9cf437a36:40243 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:02:01,749 DEBUG [M:0;1cb9cf437a36:40243 {}] master.HMaster(1795): Stopping service threads 2024-11-28T07:02:01,749 INFO [M:0;1cb9cf437a36:40243 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T07:02:01,750 INFO [M:0;1cb9cf437a36:40243 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:02:01,750 INFO [M:0;1cb9cf437a36:40243 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:02:01,750 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T07:02:01,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T07:02:01,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:02:01,759 DEBUG [M:0;1cb9cf437a36:40243 {}] zookeeper.ZKUtil(347): master:40243-0x10181890e5a0000, quorum=127.0.0.1:49831, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T07:02:01,759 WARN [M:0;1cb9cf437a36:40243 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T07:02:01,759 INFO [M:0;1cb9cf437a36:40243 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/.lastflushedseqids 2024-11-28T07:02:01,770 INFO [RS:2;1cb9cf437a36:44721 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:02:01,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:02:01,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44721-0x10181890e5a0003, quorum=127.0.0.1:49831, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:02:01,770 INFO [RS:2;1cb9cf437a36:44721 {}] regionserver.HRegionServer(1031): Exiting; stopping=1cb9cf437a36,44721,1732777312875; zookeeper connection closed. 2024-11-28T07:02:01,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741870_1046 (size=1) 2024-11-28T07:02:01,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741870_1046 (size=1) 2024-11-28T07:02:01,771 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1d4566bc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1d4566bc 2024-11-28T07:02:01,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741870_1046 (size=1) 2024-11-28T07:02:01,772 INFO [M:0;1cb9cf437a36:40243 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:02:01,772 INFO [M:0;1cb9cf437a36:40243 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T07:02:01,772 DEBUG [M:0;1cb9cf437a36:40243 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T07:02:01,773 INFO [M:0;1cb9cf437a36:40243 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:02:01,773 DEBUG [M:0;1cb9cf437a36:40243 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:02:01,773 DEBUG [M:0;1cb9cf437a36:40243 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T07:02:01,773 DEBUG [M:0;1cb9cf437a36:40243 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:02:01,773 INFO [M:0;1cb9cf437a36:40243 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.49 KB heapSize=315.39 KB 2024-11-28T07:02:01,789 DEBUG [M:0;1cb9cf437a36:40243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f230e3df3a9247c48c5895b270b547b1 is 82, key is hbase:meta,,1/info:regioninfo/1732777319813/Put/seqid=0 2024-11-28T07:02:01,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38317 is added to blk_1073741871_1047 (size=6071) 2024-11-28T07:02:01,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38193 is added to blk_1073741871_1047 (size=6071) 2024-11-28T07:02:01,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33227 is added to blk_1073741871_1047 (size=6071) 2024-11-28T07:02:01,798 INFO [M:0;1cb9cf437a36:40243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f230e3df3a9247c48c5895b270b547b1 2024-11-28T07:02:01,823 DEBUG [M:0;1cb9cf437a36:40243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36455/user/jenkins/test-data/a29d56d8-b593-9b97-75ad-602f45dc5929/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eba031480d6d42da85dda071ec98effc is 2987, key is \x00\x00\x00\x00\x00\x00\x00